2850d85b7b93f31e578520dc3b3feb24db609c62mv/*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CDDL HEADER START
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * The contents of this file are subject to the terms of the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Common Development and Distribution License (the "License").
2850d85b7b93f31e578520dc3b3feb24db609c62mv * You may not use this file except in compliance with the License.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
2850d85b7b93f31e578520dc3b3feb24db609c62mv * or http://www.opensolaris.org/os/licensing.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * See the License for the specific language governing permissions
2850d85b7b93f31e578520dc3b3feb24db609c62mv * and limitations under the License.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * When distributing Covered Code, include this CDDL HEADER in each
2850d85b7b93f31e578520dc3b3feb24db609c62mv * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If applicable, add the following below this CDDL HEADER, with the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * fields enclosed by brackets "[]" replaced with your own identifying
2850d85b7b93f31e578520dc3b3feb24db609c62mv * information: Portions Copyright [yyyy] [name of copyright owner]
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CDDL HEADER END
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Use is subject to license terms.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/thread.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/proc.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/task.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/cmn_err.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/class.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/sdt.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/atomic.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/cpu.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/clock_tick.h>
d3d50737e566cade9a08d73d2af95105ac7cd960Rafael Vanoni#include <sys/clock_impl.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <sys/sysmacros.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv#include <vm/rm.h>
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * This file contains the implementation of clock tick accounting for threads.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Every tick, user threads running on various CPUs are located and charged
2850d85b7b93f31e578520dc3b3feb24db609c62mv * with a tick to account for their use of CPU time.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Every tick, the clock() handler calls clock_tick_schedule() to perform tick
2850d85b7b93f31e578520dc3b3feb24db609c62mv * accounting for all the threads in the system. Tick accounting is done in
2850d85b7b93f31e578520dc3b3feb24db609c62mv * two phases:
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Tick scheduling Done in clock_tick_schedule(). In this phase, cross
2850d85b7b93f31e578520dc3b3feb24db609c62mv * calls are scheduled to multiple CPUs to perform
2850d85b7b93f31e578520dc3b3feb24db609c62mv * multi-threaded tick accounting. The CPUs are chosen
2850d85b7b93f31e578520dc3b3feb24db609c62mv * on a rotational basis so as to distribute the tick
2850d85b7b93f31e578520dc3b3feb24db609c62mv * accounting load evenly across all CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Tick execution Done in clock_tick_execute(). In this phase, tick
2850d85b7b93f31e578520dc3b3feb24db609c62mv * accounting is actually performed by softint handlers
2850d85b7b93f31e578520dc3b3feb24db609c62mv * on multiple CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * This implementation gives us a multi-threaded tick processing facility that
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is suitable for configurations with a large number of CPUs. On smaller
2850d85b7b93f31e578520dc3b3feb24db609c62mv * configurations it may be desirable to let the processing be single-threaded
2850d85b7b93f31e578520dc3b3feb24db609c62mv * and just allow clock() to do it as it has been done traditionally. To
2850d85b7b93f31e578520dc3b3feb24db609c62mv * facilitate this, a variable, clock_tick_threshold, is defined. Platforms
2850d85b7b93f31e578520dc3b3feb24db609c62mv * that desire multi-threading should set this variable to something
2850d85b7b93f31e578520dc3b3feb24db609c62mv * appropriate. A recommended value may be found in clock_tick.h. At boot time,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * if the number of CPUs is greater than clock_tick_threshold, multi-threading
2850d85b7b93f31e578520dc3b3feb24db609c62mv * kicks in. Note that this is a decision made at boot time. If more CPUs
2850d85b7b93f31e578520dc3b3feb24db609c62mv * are dynamically added later on to exceed the threshold, no attempt is made
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to switch to multi-threaded. Similarly, if CPUs are removed dynamically
2850d85b7b93f31e578520dc3b3feb24db609c62mv * no attempt is made to switch to single-threaded. This is to keep the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * implementation simple. Also note that the threshold can be changed for a
2850d85b7b93f31e578520dc3b3feb24db609c62mv * specific customer configuration via /etc/system.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * The boot time decision is reflected in clock_tick_single_threaded.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_threshold
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If the number of CPUs at boot time exceeds this threshold,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * multi-threaded tick accounting kicks in.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_ncpus
2850d85b7b93f31e578520dc3b3feb24db609c62mv * The number of CPUs in a set. Each set is scheduled for tick execution
2850d85b7b93f31e578520dc3b3feb24db609c62mv * on a separate processor.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_single_threaded
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Indicates whether or not tick accounting is single threaded.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_total_cpus
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Total number of online CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_cpus
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Array of online CPU pointers.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_cpu
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Per-CPU, cache-aligned data structures to facilitate multi-threading.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_active
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Counter that indicates the number of active tick processing softints
2850d85b7b93f31e578520dc3b3feb24db609c62mv * in the system.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_pending
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Number of pending ticks that need to be accounted by the softint
2850d85b7b93f31e578520dc3b3feb24db609c62mv * handlers.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_lock
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Mutex to synchronize between clock_tick_schedule() and
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU online/offline.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_cpu_id
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU id of the clock() CPU. Used to detect when the clock CPU
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is offlined.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_online_cpuset
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU set of all online processors that can be X-called.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_proc_max
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Each process is allowed to accumulate a few ticks before checking
2850d85b7b93f31e578520dc3b3feb24db609c62mv * for the task CPU time resource limit. We lower the number of calls
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to rctl_test() to make tick accounting more scalable. The tradeoff
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is that the limit may not get enforced in a timely manner. This is
2850d85b7b93f31e578520dc3b3feb24db609c62mv * typically not a problem.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_set
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Per-set structures. Each structure contains the range of CPUs
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to be processed for the set.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_nsets;
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Number of sets.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_scan
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Where to begin the scan for single-threaded mode. In multi-threaded,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the clock_tick_set itself contains a field for this.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_threshold;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_single_threaded;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_total_cpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mvcpu_t *clock_tick_cpus[NCPU];
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_cpu_t *clock_tick_cpu[NCPU];
2850d85b7b93f31e578520dc3b3feb24db609c62mvulong_t clock_tick_active;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_pending;
2850d85b7b93f31e578520dc3b3feb24db609c62mvkmutex_t clock_tick_lock;
2850d85b7b93f31e578520dc3b3feb24db609c62mvprocessorid_t clock_cpu_id;
2850d85b7b93f31e578520dc3b3feb24db609c62mvcpuset_t clock_tick_online_cpuset;
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_t clock_tick_proc_max;
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_set_t *clock_tick_set;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_nsets;
2850d85b7b93f31e578520dc3b3feb24db609c62mvint clock_tick_scan;
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataramanulong_t clock_tick_intr;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic uint_t clock_tick_execute(caddr_t, caddr_t);
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic void clock_tick_execute_common(int, int, int, clock_t, int);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv#define CLOCK_TICK_ALIGN 64 /* cache alignment */
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Clock tick initialization is done in two phases:
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * 1. Before clock_init() is called, clock_tick_init_pre() is called to set
2850d85b7b93f31e578520dc3b3feb24db609c62mv * up single-threading so the clock() can begin to do its job.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * 2. After the slave CPUs are initialized at boot time, we know the number
2850d85b7b93f31e578520dc3b3feb24db609c62mv * of CPUs. clock_tick_init_post() is called to set up multi-threading if
2850d85b7b93f31e578520dc3b3feb24db609c62mv * required.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mvvoid
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_init_pre(void)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpu_t *ctp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int i, n;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_set_t *csp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv uintptr_t buf;
2850d85b7b93f31e578520dc3b3feb24db609c62mv size_t size;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_single_threaded = 1;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv size = P2ROUNDUP(sizeof (clock_tick_cpu_t), CLOCK_TICK_ALIGN);
2850d85b7b93f31e578520dc3b3feb24db609c62mv buf = (uintptr_t)kmem_zalloc(size * NCPU + CLOCK_TICK_ALIGN, KM_SLEEP);
2850d85b7b93f31e578520dc3b3feb24db609c62mv buf = P2ROUNDUP(buf, CLOCK_TICK_ALIGN);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Perform initialization in case multi-threading is chosen later.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman if (&create_softint != NULL) {
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman clock_tick_intr = create_softint(LOCK_LEVEL,
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman clock_tick_execute, (caddr_t)NULL);
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman }
2850d85b7b93f31e578520dc3b3feb24db609c62mv for (i = 0; i < NCPU; i++, buf += size) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp = (clock_tick_cpu_t *)buf;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpu[i] = ctp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_init(&ctp->ct_lock, NULL, MUTEX_DEFAULT, NULL);
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (&create_softint != NULL) {
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman ctp->ct_intr = clock_tick_intr;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_pending = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_init(&clock_tick_lock, NULL, MUTEX_DEFAULT, NULL);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Compute clock_tick_ncpus here. We need it to compute the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * maximum number of tick sets we need to support.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv ASSERT(clock_tick_ncpus >= 0);
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_ncpus == 0)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_ncpus = CLOCK_TICK_NCPUS;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_ncpus > max_ncpus)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_ncpus = max_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Allocate and initialize the tick sets.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv n = (max_ncpus + clock_tick_ncpus - 1)/clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_set = kmem_zalloc(sizeof (clock_tick_set_t) * n, KM_SLEEP);
2850d85b7b93f31e578520dc3b3feb24db609c62mv for (i = 0; i < n; i++) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp = &clock_tick_set[i];
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_start = i * clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_scan = csp->ct_start;
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_end = csp->ct_start;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvvoid
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_init_post(void)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If a platform does not provide create_softint() and invoke_softint(),
2850d85b7b93f31e578520dc3b3feb24db609c62mv * then we assume single threaded.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (&invoke_softint == NULL)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_threshold = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv ASSERT(clock_tick_threshold >= 0);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_threshold == 0)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_threshold = max_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If a platform does not specify a threshold or if the number of CPUs
2850d85b7b93f31e578520dc3b3feb24db609c62mv * at boot time does not exceed the threshold, tick accounting remains
2850d85b7b93f31e578520dc3b3feb24db609c62mv * single-threaded.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (ncpus <= clock_tick_threshold) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_ncpus = max_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_proc_max = 1;
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * OK. Multi-thread tick processing. If a platform has not specified
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the CPU set size for multi-threading, then use the default value.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * This value has been arrived through measurements on large
2850d85b7b93f31e578520dc3b3feb24db609c62mv * configuration systems.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_single_threaded = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_proc_max == 0) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_proc_max = CLOCK_TICK_PROC_MAX;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (hires_tick)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_proc_max *= 10;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic void
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_schedule_one(clock_tick_set_t *csp, int pending, processorid_t cid)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpu_t *ctp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv ASSERT(&invoke_softint != NULL);
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman atomic_inc_ulong(&clock_tick_active);
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Schedule tick accounting for a set of CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp = clock_tick_cpu[cid];
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(&ctp->ct_lock);
1b7f7204f8aa806a57b84cdeba100d819be85a61Rafael Vanoni ctp->ct_lbolt = LBOLT_NO_ACCOUNT;
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_pending += pending;
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_start = csp->ct_start;
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_end = csp->ct_end;
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_scan = csp->ct_scan;
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&ctp->ct_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv invoke_softint(cid, ctp->ct_intr);
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Return without waiting for the softint to finish.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic void
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_process(cpu_t *cp, clock_t mylbolt, int pending)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv kthread_t *t;
2850d85b7b93f31e578520dc3b3feb24db609c62mv kmutex_t *plockp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int notick, intr;
2850d85b7b93f31e578520dc3b3feb24db609c62mv klwp_id_t lwp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * The locking here is rather tricky. thread_free_prevent()
2850d85b7b93f31e578520dc3b3feb24db609c62mv * prevents the thread returned from being freed while we
2850d85b7b93f31e578520dc3b3feb24db609c62mv * are looking at it. We can then check if the thread
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is exiting and get the appropriate p_lock if it
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is not. We have to be careful, though, because
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the _process_ can still be freed while we've
2850d85b7b93f31e578520dc3b3feb24db609c62mv * prevented thread free. To avoid touching the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * proc structure we put a pointer to the p_lock in the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * thread structure. The p_lock is persistent so we
2850d85b7b93f31e578520dc3b3feb24db609c62mv * can acquire it even if the process is gone. At that
2850d85b7b93f31e578520dc3b3feb24db609c62mv * point we can check (again) if the thread is exiting
2850d85b7b93f31e578520dc3b3feb24db609c62mv * and either drop the lock or do the tick processing.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv t = cp->cpu_thread; /* Current running thread */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (CPU == cp) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * 't' will be the tick processing thread on this
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU. Use the pinned thread (if any) on this CPU
2850d85b7b93f31e578520dc3b3feb24db609c62mv * as the target of the clock tick.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (t->t_intr != NULL)
2850d85b7b93f31e578520dc3b3feb24db609c62mv t = t->t_intr;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * We use thread_free_prevent to keep the currently running
2850d85b7b93f31e578520dc3b3feb24db609c62mv * thread from being freed or recycled while we're
2850d85b7b93f31e578520dc3b3feb24db609c62mv * looking at it.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_prevent(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * We cannot hold the cpu_lock to prevent the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * cpu_active from changing in the clock interrupt.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * As long as we don't block (or don't get pre-empted)
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the cpu_list will not change (all threads are paused
2850d85b7b93f31e578520dc3b3feb24db609c62mv * before list modification).
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (CLOCK_TICK_CPU_OFFLINE(cp)) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Make sure the thread is still on the CPU.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if ((t != cp->cpu_thread) &&
2850d85b7b93f31e578520dc3b3feb24db609c62mv ((cp != CPU) || (t != cp->cpu_thread->t_intr))) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * We could not locate the thread. Skip this CPU. Race
2850d85b7b93f31e578520dc3b3feb24db609c62mv * conditions while performing these checks are benign.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * These checks are not perfect and they don't need
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to be.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv intr = t->t_flag & T_INTR_THREAD;
2850d85b7b93f31e578520dc3b3feb24db609c62mv lwp = ttolwp(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (lwp == NULL || (t->t_proc_flag & TP_LWPEXIT) || intr) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Thread is exiting (or uninteresting) so don't
2850d85b7b93f31e578520dc3b3feb24db609c62mv * do tick processing.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * OK, try to grab the process lock. See
2850d85b7b93f31e578520dc3b3feb24db609c62mv * comments above for why we're not using
2850d85b7b93f31e578520dc3b3feb24db609c62mv * ttoproc(t)->p_lockp here.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv plockp = t->t_plockp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(plockp);
2850d85b7b93f31e578520dc3b3feb24db609c62mv /* See above comment. */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (CLOCK_TICK_CPU_OFFLINE(cp)) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(plockp);
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * The thread may have exited between when we
2850d85b7b93f31e578520dc3b3feb24db609c62mv * checked above, and when we got the p_lock.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (t->t_proc_flag & TP_LWPEXIT) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(plockp);
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Either we have the p_lock for the thread's process,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * or we don't care about the thread structure any more.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Either way we can allow thread free.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv thread_free_allow(t);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If we haven't done tick processing for this
2850d85b7b93f31e578520dc3b3feb24db609c62mv * lwp, then do it now. Since we don't hold the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * lwp down on a CPU it can migrate and show up
2850d85b7b93f31e578520dc3b3feb24db609c62mv * more than once, hence the lbolt check. mylbolt
2850d85b7b93f31e578520dc3b3feb24db609c62mv * is copied at the time of tick scheduling to prevent
2850d85b7b93f31e578520dc3b3feb24db609c62mv * lbolt mismatches.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Also, make sure that it's okay to perform the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * tick processing before calling clock_tick.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Setting notick to a TRUE value (ie. not 0)
2850d85b7b93f31e578520dc3b3feb24db609c62mv * results in tick processing not being performed for
2850d85b7b93f31e578520dc3b3feb24db609c62mv * that thread.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv notick = ((cp->cpu_flags & CPU_QUIESCED) || CPU_ON_INTR(cp) ||
2850d85b7b93f31e578520dc3b3feb24db609c62mv (cp->cpu_dispthread == cp->cpu_idle_thread));
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv if ((!notick) && (t->t_lbolt < mylbolt)) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv t->t_lbolt = mylbolt;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick(t, pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(plockp);
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvvoid
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_schedule(int one_sec)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv ulong_t active;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int i, end;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_set_t *csp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv cpu_t *cp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_cpu_id != CPU->cpu_id)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_cpu_id = CPU->cpu_id;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_single_threaded) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Each tick cycle, start the scan from a different
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU for the sake of fairness.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv end = clock_tick_total_cpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_scan++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (clock_tick_scan >= end)
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_scan = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
d3d50737e566cade9a08d73d2af95105ac7cd960Rafael Vanoni clock_tick_execute_common(0, clock_tick_scan, end,
1b7f7204f8aa806a57b84cdeba100d819be85a61Rafael Vanoni LBOLT_NO_ACCOUNT, 1);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If the previous invocation of handlers is not yet finished, then
2850d85b7b93f31e578520dc3b3feb24db609c62mv * simply increment a pending count and return. Eventually when they
2850d85b7b93f31e578520dc3b3feb24db609c62mv * finish, the pending count is passed down to the next set of
2850d85b7b93f31e578520dc3b3feb24db609c62mv * handlers to process. This way, ticks that have already elapsed
2850d85b7b93f31e578520dc3b3feb24db609c62mv * in the past are handled as quickly as possible to minimize the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * chances of threads getting away before their pending ticks are
2850d85b7b93f31e578520dc3b3feb24db609c62mv * accounted. The other benefit is that if the pending count is
2850d85b7b93f31e578520dc3b3feb24db609c62mv * more than one, it can be handled by a single invocation of
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick(). This is a good optimization for large configuration
2850d85b7b93f31e578520dc3b3feb24db609c62mv * busy systems where tick accounting can get backed up for various
2850d85b7b93f31e578520dc3b3feb24db609c62mv * reasons.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_pending++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv active = clock_tick_active;
2850d85b7b93f31e578520dc3b3feb24db609c62mv active = atomic_cas_ulong(&clock_tick_active, active, active);
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (active)
2850d85b7b93f31e578520dc3b3feb24db609c62mv return;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * We want to handle the clock CPU here. If we
2850d85b7b93f31e578520dc3b3feb24db609c62mv * scheduled the accounting for the clock CPU to another
2850d85b7b93f31e578520dc3b3feb24db609c62mv * processor, that processor will find only the clock() thread
2850d85b7b93f31e578520dc3b3feb24db609c62mv * running and not account for any user thread below it. Also,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * we want to handle this before we block on anything and allow
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the pinned thread below the current thread to escape.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
1b7f7204f8aa806a57b84cdeba100d819be85a61Rafael Vanoni clock_tick_process(CPU, LBOLT_NO_ACCOUNT, clock_tick_pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(&clock_tick_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Schedule each set on a separate processor.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = clock_cpu_list;
2850d85b7b93f31e578520dc3b3feb24db609c62mv for (i = 0; i < clock_tick_nsets; i++) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp = &clock_tick_set[i];
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Pick the next online CPU in list for scheduling tick
2850d85b7b93f31e578520dc3b3feb24db609c62mv * accounting. The clock_tick_lock is held by the caller.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * So, CPU online/offline cannot muck with this while
2850d85b7b93f31e578520dc3b3feb24db609c62mv * we are picking our CPU to X-call.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (cp == CPU)
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = cp->cpu_next_onln;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Each tick cycle, start the scan from a different
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU for the sake of fairness.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_scan++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (csp->ct_scan >= csp->ct_end)
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_scan = csp->ct_start;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_schedule_one(csp, clock_tick_pending, cp->cpu_id);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = cp->cpu_next_onln;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (one_sec) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Move the CPU pointer around every second. This is so
2850d85b7b93f31e578520dc3b3feb24db609c62mv * all the CPUs can be X-called in a round-robin fashion
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to evenly distribute the X-calls. We don't do this
2850d85b7b93f31e578520dc3b3feb24db609c62mv * at a faster rate than this because we don't want
2850d85b7b93f31e578520dc3b3feb24db609c62mv * to affect cache performance negatively.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_cpu_list = clock_cpu_list->cpu_next_onln;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&clock_tick_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_pending = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic void
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_execute_common(int start, int scan, int end, clock_t mylbolt,
2850d85b7b93f31e578520dc3b3feb24db609c62mv int pending)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv cpu_t *cp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int i;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv ASSERT((start <= scan) && (scan <= end));
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Handle the thread on current CPU first. This is to prevent a
2850d85b7b93f31e578520dc3b3feb24db609c62mv * pinned thread from escaping if we ever block on something.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Note that in the single-threaded mode, this handles the clock
2850d85b7b93f31e578520dc3b3feb24db609c62mv * CPU.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_process(CPU, mylbolt, pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Perform tick accounting for the threads running on
2850d85b7b93f31e578520dc3b3feb24db609c62mv * the scheduled CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv for (i = scan; i < end; i++) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = clock_tick_cpus[i];
2850d85b7b93f31e578520dc3b3feb24db609c62mv if ((cp == NULL) || (cp == CPU) || (cp->cpu_id == clock_cpu_id))
2850d85b7b93f31e578520dc3b3feb24db609c62mv continue;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_process(cp, mylbolt, pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv for (i = start; i < scan; i++) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = clock_tick_cpus[i];
2850d85b7b93f31e578520dc3b3feb24db609c62mv if ((cp == NULL) || (cp == CPU) || (cp->cpu_id == clock_cpu_id))
2850d85b7b93f31e578520dc3b3feb24db609c62mv continue;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_process(cp, mylbolt, pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*ARGSUSED*/
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic uint_t
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_execute(caddr_t arg1, caddr_t arg2)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpu_t *ctp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int start, scan, end, pending;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_t mylbolt;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * We could have raced with cpu offline. We don't want to
2850d85b7b93f31e578520dc3b3feb24db609c62mv * process anything on an offlined CPU. If we got blocked
2850d85b7b93f31e578520dc3b3feb24db609c62mv * on anything, we may not get scheduled when we wakeup
2850d85b7b93f31e578520dc3b3feb24db609c62mv * later on.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (!CLOCK_TICK_XCALL_SAFE(CPU))
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman goto out;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
07247649496653b0ece32ce7ea710057f85fa9daMadhavan Venkataraman ctp = clock_tick_cpu[CPU->cpu_id];
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(&ctp->ct_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv pending = ctp->ct_pending;
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (pending == 0) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * If a CPU is busy at LOCK_LEVEL, then an invocation
2850d85b7b93f31e578520dc3b3feb24db609c62mv * of this softint may be queued for some time. In that case,
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_active will not be incremented.
2850d85b7b93f31e578520dc3b3feb24db609c62mv * clock_tick_schedule() will then assume that the previous
2850d85b7b93f31e578520dc3b3feb24db609c62mv * invocation is done and post a new softint. The first one
2850d85b7b93f31e578520dc3b3feb24db609c62mv * that gets in will reset the pending count so the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * second one is a noop.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&ctp->ct_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv goto out;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv ctp->ct_pending = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv start = ctp->ct_start;
2850d85b7b93f31e578520dc3b3feb24db609c62mv end = ctp->ct_end;
2850d85b7b93f31e578520dc3b3feb24db609c62mv scan = ctp->ct_scan;
2850d85b7b93f31e578520dc3b3feb24db609c62mv mylbolt = ctp->ct_lbolt;
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&ctp->ct_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_execute_common(start, scan, end, mylbolt, pending);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvout:
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Signal completion to the clock handler.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv atomic_dec_ulong(&clock_tick_active);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv return (1);
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv/*ARGSUSED*/
2850d85b7b93f31e578520dc3b3feb24db609c62mvstatic int
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_cpu_setup(cpu_setup_t what, int cid, void *arg)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv cpu_t *cp, *ncp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv int i, set;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_set_t *csp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv /*
2850d85b7b93f31e578520dc3b3feb24db609c62mv * This function performs some computations at CPU offline/online
2850d85b7b93f31e578520dc3b3feb24db609c62mv * time. The computed values are used during tick scheduling and
2850d85b7b93f31e578520dc3b3feb24db609c62mv * execution phases. This avoids having to compute things on
2850d85b7b93f31e578520dc3b3feb24db609c62mv * an every tick basis. The other benefit is that we perform the
2850d85b7b93f31e578520dc3b3feb24db609c62mv * computations only for onlined CPUs (not offlined ones). As a
2850d85b7b93f31e578520dc3b3feb24db609c62mv * result, no tick processing is attempted for offlined CPUs.
2850d85b7b93f31e578520dc3b3feb24db609c62mv *
2850d85b7b93f31e578520dc3b3feb24db609c62mv * Also, cpu_offline() calls this function before checking for
2850d85b7b93f31e578520dc3b3feb24db609c62mv * active interrupt threads. This allows us to avoid posting
2850d85b7b93f31e578520dc3b3feb24db609c62mv * cross calls to CPUs that are being offlined.
2850d85b7b93f31e578520dc3b3feb24db609c62mv */
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = cpu[cid];
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(&clock_tick_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv switch (what) {
2850d85b7b93f31e578520dc3b3feb24db609c62mv case CPU_ON:
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpus[clock_tick_total_cpus] = cp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv set = clock_tick_total_cpus / clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp = &clock_tick_set[set];
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_end++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_total_cpus++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_nsets =
2850d85b7b93f31e578520dc3b3feb24db609c62mv (clock_tick_total_cpus + clock_tick_ncpus - 1) /
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv CPUSET_ADD(clock_tick_online_cpuset, cp->cpu_id);
2850d85b7b93f31e578520dc3b3feb24db609c62mv membar_sync();
2850d85b7b93f31e578520dc3b3feb24db609c62mv break;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv case CPU_OFF:
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (&sync_softint != NULL)
2850d85b7b93f31e578520dc3b3feb24db609c62mv sync_softint(clock_tick_online_cpuset);
2850d85b7b93f31e578520dc3b3feb24db609c62mv CPUSET_DEL(clock_tick_online_cpuset, cp->cpu_id);
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_total_cpus--;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpus[clock_tick_total_cpus] = NULL;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_nsets =
2850d85b7b93f31e578520dc3b3feb24db609c62mv (clock_tick_total_cpus + clock_tick_ncpus - 1) /
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv set = clock_tick_total_cpus / clock_tick_ncpus;
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp = &clock_tick_set[set];
2850d85b7b93f31e578520dc3b3feb24db609c62mv csp->ct_end--;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv i = 0;
2850d85b7b93f31e578520dc3b3feb24db609c62mv ncp = cpu_active;
2850d85b7b93f31e578520dc3b3feb24db609c62mv do {
2850d85b7b93f31e578520dc3b3feb24db609c62mv if (cp == ncp)
2850d85b7b93f31e578520dc3b3feb24db609c62mv continue;
2850d85b7b93f31e578520dc3b3feb24db609c62mv clock_tick_cpus[i] = ncp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv i++;
2850d85b7b93f31e578520dc3b3feb24db609c62mv } while ((ncp = ncp->cpu_next_onln) != cpu_active);
2850d85b7b93f31e578520dc3b3feb24db609c62mv ASSERT(i == clock_tick_total_cpus);
2850d85b7b93f31e578520dc3b3feb24db609c62mv membar_sync();
2850d85b7b93f31e578520dc3b3feb24db609c62mv break;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv default:
2850d85b7b93f31e578520dc3b3feb24db609c62mv break;
2850d85b7b93f31e578520dc3b3feb24db609c62mv }
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&clock_tick_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv return (0);
2850d85b7b93f31e578520dc3b3feb24db609c62mv}
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mvvoid
2850d85b7b93f31e578520dc3b3feb24db609c62mvclock_tick_mp_init(void)
2850d85b7b93f31e578520dc3b3feb24db609c62mv{
2850d85b7b93f31e578520dc3b3feb24db609c62mv cpu_t *cp;
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_enter(&cpu_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv cp = cpu_active;
2850d85b7b93f31e578520dc3b3feb24db609c62mv do {
2850d85b7b93f31e578520dc3b3feb24db609c62mv (void) clock_tick_cpu_setup(CPU_ON, cp->cpu_id, NULL);
2850d85b7b93f31e578520dc3b3feb24db609c62mv } while ((cp = cp->cpu_next_onln) != cpu_active);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv register_cpu_setup_func(clock_tick_cpu_setup, NULL);
2850d85b7b93f31e578520dc3b3feb24db609c62mv
2850d85b7b93f31e578520dc3b3feb24db609c62mv mutex_exit(&cpu_lock);
2850d85b7b93f31e578520dc3b3feb24db609c62mv}