lgrpplat.c revision 2dae3fb5f236a83380b9deea54417c4e1f535121
080575042aba2197b425ebfd52061dea061a9aa1xy * CDDL HEADER START
080575042aba2197b425ebfd52061dea061a9aa1xy * The contents of this file are subject to the terms of the
080575042aba2197b425ebfd52061dea061a9aa1xy * Common Development and Distribution License, Version 1.0 only
080575042aba2197b425ebfd52061dea061a9aa1xy * (the "License"). You may not use this file except in compliance
080575042aba2197b425ebfd52061dea061a9aa1xy * with the License.
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
080575042aba2197b425ebfd52061dea061a9aa1xy * See the License for the specific language governing permissions
080575042aba2197b425ebfd52061dea061a9aa1xy * and limitations under the License.
080575042aba2197b425ebfd52061dea061a9aa1xy * When distributing Covered Code, include this CDDL HEADER in each
080575042aba2197b425ebfd52061dea061a9aa1xy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
080575042aba2197b425ebfd52061dea061a9aa1xy * If applicable, add the following below this CDDL HEADER, with the
080575042aba2197b425ebfd52061dea061a9aa1xy * fields enclosed by brackets "[]" replaced with your own identifying
080575042aba2197b425ebfd52061dea061a9aa1xy * information: Portions Copyright [yyyy] [name of copyright owner]
080575042aba2197b425ebfd52061dea061a9aa1xy * CDDL HEADER END
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * Use is subject to license terms.
080575042aba2197b425ebfd52061dea061a9aa1xy#pragma ident "%Z%%M% %I% %E% SMI"
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/pci_impl.h> /* for PCI configuration space macros */
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/x86_archext.h> /* for x86_feature and X86_AMD */
080575042aba2197b425ebfd52061dea061a9aa1xy * lgroup platform support for x86 platforms.
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_CPU_TO_NODE(cpu) (chip_plat_get_chipid(cpu))
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */
080575042aba2197b425ebfd52061dea061a9aa1xy * Multiprocessor Opteron machines have Non Uniform Memory Access (NUMA).
080575042aba2197b425ebfd52061dea061a9aa1xy * Until System Affinity Resource Table (SRAT) becomes part of ACPI standard,
080575042aba2197b425ebfd52061dea061a9aa1xy * we need to examine registers in PCI configuration space to determine how
080575042aba2197b425ebfd52061dea061a9aa1xy * many nodes are in the system and which CPUs and memory are in each node.
080575042aba2197b425ebfd52061dea061a9aa1xy * This could be determined by probing all memory from each CPU, but that is
c7770590c6dc06be2588a5c21427e2d823baa989mx * too expensive to do while booting the kernel.
080575042aba2197b425ebfd52061dea061a9aa1xy * NOTE: Using these PCI configuration space registers to determine this
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * locality info is Opteron K8 specific and not guaranteed to work on
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * the next generation Opteron processor. Furthermore, we assume that
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * there is one CPU per node and CPU 0 is in node 0, CPU 1 is in node 1,
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * etc. which should be true for Opteron K8....
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * Opteron DRAM Address Map in PCI configuration space gives base and limit
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * of physical memory in each node for Opteron K8. The following constants
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * and macros define their contents, structure, and access.
080575042aba2197b425ebfd52061dea061a9aa1xy * How many bits to shift Opteron DRAM Address Map base and limit registers
080575042aba2197b425ebfd52061dea061a9aa1xy * to get actual value
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMADDR_LSHIFT_ADDR 8 /* shift left for address */
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMADDR_MASK_OFF 0xFFFFFF /* offset for address */
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems * Bit masks defining what's in Opteron DRAM Address Map base register
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China#define OPT_DRAMBASE_MASK_RE 0x1 /* read enable */
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China#define OPT_DRAMBASE_MASK_INTRLVEN 0x700 /* interleave */
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMBASE_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */
080575042aba2197b425ebfd52061dea061a9aa1xy * Macros to get values from Opteron DRAM Address Map base register
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Bit masks defining what's in Opteron DRAM Address Map limit register
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_DSTNODE 0x7 /* destination node */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_INTRLVSEL 0x70 /* interleave select */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_ADDR 0xFFFF0000 /* addr bits 39-24 */
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Macros to get values from Opteron DRAM Address Map limit register
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China OPT_DRAMADDR_LSHIFT_ADDR)
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Opteron Node ID register in PCI configuration space contains
25f2d433de915875c8393f0b0dc14aa155997ad0xy * number of nodes in system, etc. for Opteron K8. The following
25f2d433de915875c8393f0b0dc14aa155997ad0xy * constants and macros define its contents, structure, and access.
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Bit masks defining what's in Opteron Node ID register
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China#define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * How many bits in Opteron Node ID register to shift right to get actual value
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China#define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Macros to get values from Opteron Node ID register
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * PCI configuration space registers accessed by specifying
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * a bus, device, function, and offset. The following constants
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * define the values needed to access Opteron K8 configuration
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * info to determine its node topology
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Opteron PCI configuration space register function values
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */
25f2d433de915875c8393f0b0dc14aa155997ad0xy * PCI Configuration Space register offsets
c7770590c6dc06be2588a5c21427e2d823baa989mx#define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_PCS_OFF_DRAMBASE 0x40 /* DRAM Base register (node 0) */
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron PCI Configuration Space device IDs for nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */
080575042aba2197b425ebfd52061dea061a9aa1xy * Bookkeeping for latencies seen during probing (used for verification)
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystemstypedef struct lgrp_plat_latency_acct {
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems hrtime_t la_value; /* latency value */
080575042aba2197b425ebfd52061dea061a9aa1xy * Choices for probing to determine lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xytypedef enum lgrp_plat_probe_op {
080575042aba2197b425ebfd52061dea061a9aa1xy LGRP_PLAT_PROBE_VENDOR /* Read vendor ID on Northbridge */
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron DRAM address map gives base and limit for physical memory in a node
c7770590c6dc06be2588a5c21427e2d823baa989mxtypedef struct opt_dram_addr_map {
080575042aba2197b425ebfd52061dea061a9aa1xy * Starting and ending page for physical memory in node
080575042aba2197b425ebfd52061dea061a9aa1xytypedef struct phys_addr_map {
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron DRAM address map for each node
080575042aba2197b425ebfd52061dea061a9aa1xy * Node ID register contents for each node
080575042aba2197b425ebfd52061dea061a9aa1xy * Whether memory is interleaved across nodes causing MPO to be disabled
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Number of nodes in system
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Physical address range for memory in each node
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * Probe costs (individual and total) and flush cost
080575042aba2197b425ebfd52061dea061a9aa1xy * Error code for latency adjustment and verification
080575042aba2197b425ebfd52061dea061a9aa1xy * How much latencies were off from minimum values gotten
080575042aba2197b425ebfd52061dea061a9aa1xy * Unique probe latencies and number of occurrences of each
080575042aba2197b425ebfd52061dea061a9aa1xylgrp_plat_latency_acct_t lgrp_plat_probe_lat_acct[MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy * Size of memory buffer in each node for probing
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc * Virtual address of page in each node for probing
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystemscaddr_t lgrp_plat_probe_memory[MAX_NODES];
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc * Number of unique latencies in probe times
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China * How many rounds of probing to do
080575042aba2197b425ebfd52061dea061a9aa1xy * Number of samples to take when probing each node
080575042aba2197b425ebfd52061dea061a9aa1xy * How to probe to determine lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xylgrp_plat_probe_op_t lgrp_plat_probe_op = LGRP_PLAT_PROBE_VENDOR;
080575042aba2197b425ebfd52061dea061a9aa1xy * PFN of page in each node for probing
080575042aba2197b425ebfd52061dea061a9aa1xy * Whether probe time was suspect (ie. not within tolerance of value that it
080575042aba2197b425ebfd52061dea061a9aa1xy * should match)
ea6b684a18957883cb91b3d22a9d989f986e5a32yy * How long it takes to access memory from each node
c7770590c6dc06be2588a5c21427e2d823baa989mx * Min and max node memory probe times seen
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_time_max = 0;
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_time_min = -1;
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_max[MAX_NODES][MAX_NODES];
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_min[MAX_NODES][MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy * Allocate lgrp and lgrp stat arrays statically.
080575042aba2197b425ebfd52061dea061a9aa1xy * Determine whether we're running on an AMD Opteron K8 machine
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy return (1);
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy return ((int)hand);
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy return (-1);
080575042aba2197b425ebfd52061dea061a9aa1xy * Configure memory nodes for machines with more than one node (ie NUMA)
080575042aba2197b425ebfd52061dea061a9aa1xy pfn_t cur_start, cur_end; /* start & end addr of subrange */
080575042aba2197b425ebfd52061dea061a9aa1xy * Boot install lists are arranged <addr, len>, ...
080575042aba2197b425ebfd52061dea061a9aa1xy * When there is only one memnode, just add memory to memnode
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * mem_node_add_slice() expects to get a memory range that
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * is within one memnode, so need to split any memory range
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * that spans multiple memnodes into subranges that are each
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * contained within one memnode when feeding them to
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * mem_node_add_slice()
25f2d433de915875c8393f0b0dc14aa155997ad0xy * End of current subrange should not span memnodes
080575042aba2197b425ebfd52061dea061a9aa1xy * Next subrange starts after end of current one
080575042aba2197b425ebfd52061dea061a9aa1xy * Platform-specific initialization of lgroups
080575042aba2197b425ebfd52061dea061a9aa1xy * Initialize as a UMA machine if this isn't an Opteron
080575042aba2197b425ebfd52061dea061a9aa1xy * Read configuration registers from PCI configuration space to
080575042aba2197b425ebfd52061dea061a9aa1xy * determine node information, which memory is in each node, etc.
080575042aba2197b425ebfd52061dea061a9aa1xy * Write to PCI configuration space address register to specify
080575042aba2197b425ebfd52061dea061a9aa1xy * which configuration register to read and read/write PCI
ea6b684a18957883cb91b3d22a9d989f986e5a32yy * configuration space data register to get/set contents
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Read node ID register for node 0 to get node count
080575042aba2197b425ebfd52061dea061a9aa1xy * Read node ID register (except for node 0 which we just read)
080575042aba2197b425ebfd52061dea061a9aa1xy if (node > 0) {
080575042aba2197b425ebfd52061dea061a9aa1xy * Read DRAM base and limit registers which specify
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China * physical memory range of each node
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China outl(PCI_CONFADD, PCI_CADDR1(bus, dev, OPT_PCS_FUNC_ADDRMAP,
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China opt_dram_map[node].base = inl(PCI_CONFDATA);
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China if (opt_dram_map[node].base & OPT_DRAMBASE_MASK_INTRLVEN)
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China lgrp_plat_mem_intrlv++;
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China off += 4; /* limit register offset */
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China outl(PCI_CONFADD, PCI_CADDR1(bus, dev, OPT_PCS_FUNC_ADDRMAP,
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China opt_dram_map[node].limit = inl(PCI_CONFDATA);
080575042aba2197b425ebfd52061dea061a9aa1xy * Increment device number to next node and register offset for
25f2d433de915875c8393f0b0dc14aa155997ad0xy * DRAM base register of next node
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Get PFN for first page in each node,
25f2d433de915875c8393f0b0dc14aa155997ad0xy * so we can probe memory to determine latency topology
080575042aba2197b425ebfd52061dea061a9aa1xy * Remember physical address range of each node for use later
080575042aba2197b425ebfd52061dea061a9aa1xy * Only use one memory node if memory is interleaved between any nodes
080575042aba2197b425ebfd52061dea061a9aa1xy * Probing errors can mess up the lgroup topology and force us
25f2d433de915875c8393f0b0dc14aa155997ad0xy * fall back to a 2 level lgroup topology. Here we bound how
25f2d433de915875c8393f0b0dc14aa155997ad0xy * tall the lgroup topology can grow in hopes of avoiding any
080575042aba2197b425ebfd52061dea061a9aa1xy * anamolies in probing from messing up the lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xy * by limiting the accuracy of the latency topology.
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Assume that nodes will at least be configured in a ring,
080575042aba2197b425ebfd52061dea061a9aa1xy * so limit height of lgroup topology to be less than number
080575042aba2197b425ebfd52061dea061a9aa1xy * of nodes on a system with 4 or more nodes
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Lgroups on Opteron architectures have but a single physical
080575042aba2197b425ebfd52061dea061a9aa1xy * processor. Tune lgrp_expand_proc_thresh and lgrp_expand_proc_diff
080575042aba2197b425ebfd52061dea061a9aa1xy * so that lgrp_choose() will spread things out aggressively.
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to
080575042aba2197b425ebfd52061dea061a9aa1xy * be considered same
080575042aba2197b425ebfd52061dea061a9aa1xy * Adjust latencies between nodes to be symmetric, normalize latencies between
080575042aba2197b425ebfd52061dea061a9aa1xy * any nodes that are within some tolerance to be same, and make local
25f2d433de915875c8393f0b0dc14aa155997ad0xy * latencies be same
c7770590c6dc06be2588a5c21427e2d823baa989mxstatic void
080575042aba2197b425ebfd52061dea061a9aa1xy const lgrp_config_flag_t cflag = LGRP_CONFIG_LATENCY_CHANGE;
080575042aba2197b425ebfd52061dea061a9aa1xy * Nothing to do when this is an UMA machine
080575042aba2197b425ebfd52061dea061a9aa1xy * Make sure that latencies are symmetric between any two nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy * (ie. latency(node0, node1) == latency(node1, node0))
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystems t2 = lgrp_plat_probe_times[j][i];
080575042aba2197b425ebfd52061dea061a9aa1xy * Latencies should be same
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Use minimum of two latencies which should be same
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Track suspect probe times not within tolerance of
25f2d433de915875c8393f0b0dc14aa155997ad0xy * min value
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Remember how much values are corrected by
080575042aba2197b425ebfd52061dea061a9aa1xy * Keep track of which latencies get corrected
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < MAX_NODES; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < MAX_NODES; j++)
080575042aba2197b425ebfd52061dea061a9aa1xy * For every two nodes, see whether there is another pair of nodes which
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * are about the same distance apart and make the latencies be the same
080575042aba2197b425ebfd52061dea061a9aa1xy * if they are close enough together
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
080575042aba2197b425ebfd52061dea061a9aa1xy * Pick one pair of nodes (i, j)
080575042aba2197b425ebfd52061dea061a9aa1xy * and get latency between them
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip this pair of nodes if there isn't a latency
080575042aba2197b425ebfd52061dea061a9aa1xy * for it yet
47b7744cbea59975a6b583125b7ed1ff2ac45313yy if (t1 == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy for (k = 0; k < lgrp_plat_node_cnt; k++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (l = 0; l < lgrp_plat_node_cnt; l++) {
080575042aba2197b425ebfd52061dea061a9aa1xy * Pick another pair of nodes (k, l)
080575042aba2197b425ebfd52061dea061a9aa1xy * not same as (i, j) and get latency
080575042aba2197b425ebfd52061dea061a9aa1xy * between them
080575042aba2197b425ebfd52061dea061a9aa1xy if (k == i && l == j)
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip this pair of nodes if there
080575042aba2197b425ebfd52061dea061a9aa1xy * isn't a latency for it yet
080575042aba2197b425ebfd52061dea061a9aa1xy if (t2 == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip nodes (k, l) if they already
25f2d433de915875c8393f0b0dc14aa155997ad0xy * have same latency as (i, j) or
080575042aba2197b425ebfd52061dea061a9aa1xy * their latency isn't close enough to
080575042aba2197b425ebfd52061dea061a9aa1xy * be considered/made the same
080575042aba2197b425ebfd52061dea061a9aa1xy * Make latency(i, j) same as
25f2d433de915875c8393f0b0dc14aa155997ad0xy * latency(k, l), try to use latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy * that has been adjusted already to get
25f2d433de915875c8393f0b0dc14aa155997ad0xy * more consistency (if possible), and
25f2d433de915875c8393f0b0dc14aa155997ad0xy * remember which latencies were
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * adjusted for next time
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (lat_corrected[i][j]) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy } else if (lat_corrected[k][l]) {
080575042aba2197b425ebfd52061dea061a9aa1xy * Local latencies should be same
080575042aba2197b425ebfd52061dea061a9aa1xy * - Find min and max local latencies
080575042aba2197b425ebfd52061dea061a9aa1xy * - Make all local latencies be minimum
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++) {
080575042aba2197b425ebfd52061dea061a9aa1xy if (t == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Track suspect probe times that aren't within
25f2d433de915875c8393f0b0dc14aa155997ad0xy * tolerance of minimum local latency and how much
25f2d433de915875c8393f0b0dc14aa155997ad0xy * probe times are corrected by
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems lgrp_plat_probe_errors[i][i] += local - min;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Make local latencies be minimum
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems lgrp_plat_probe_times[i][i] = min;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * Determine max probe time again since just adjusted latencies
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng for (i = 0; i < lgrp_plat_node_cnt; i++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Verify following about latencies between nodes:
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a))
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Local latencies same
080575042aba2197b425ebfd52061dea061a9aa1xy * - Local < remote
080575042aba2197b425ebfd52061dea061a9aa1xy * - Number of latencies seen is reasonable
080575042aba2197b425ebfd52061dea061a9aa1xy * - Number of occurrences of a given latency should be more than 1
080575042aba2197b425ebfd52061dea061a9aa1xy * Returns:
080575042aba2197b425ebfd52061dea061a9aa1xy * 0 Success
080575042aba2197b425ebfd52061dea061a9aa1xy * -1 Not symmetric
25f2d433de915875c8393f0b0dc14aa155997ad0xy * -2 Local latencies not same
080575042aba2197b425ebfd52061dea061a9aa1xy * -3 Local >= remote
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy * -4 Wrong number of latencies
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * -5 Not enough occurrences of given latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Nothing to do when this is an UMA machine, lgroup topology is
25f2d433de915875c8393f0b0dc14aa155997ad0xy * limited to 2 levels, or there aren't any probe times yet
25f2d433de915875c8393f0b0dc14aa155997ad0xy (lgrp_plat_probe_time_max == 0 && lgrp_plat_probe_time_min == -1))
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Make sure that latencies are symmetric between any two nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy * (ie. latency(node0, node1) == latency(node1, node0))
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner for (j = 0; j < lgrp_plat_node_cnt; j++) {
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner * Local latencies should be same
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Local latencies should be less than remote
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (i == j || t2 == 0)
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Rest of checks are not very useful for machines with less than
25f2d433de915875c8393f0b0dc14aa155997ad0xy * 4 nodes (which means less than 3 latencies on Opteron)
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China * Need to see whether done probing in order to verify number of
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China * latencies are correct
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (i = 0; i < lgrp_plat_node_cnt; i++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
7941757c1241fe30e30f921910595c8ac6af9ef1xy * Determine number of unique latencies seen in probe times,
25f2d433de915875c8393f0b0dc14aa155997ad0xy * their values, and number of occurrences of each
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng for (i = 0; i < lgrp_plat_node_cnt; i++) {
47b7744cbea59975a6b583125b7ed1ff2ac45313yy for (j = 0; j < lgrp_plat_node_cnt; j++) {
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Look at each probe time
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1 == 0)
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Account for unique latencies
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China for (k = 0; k < lgrp_plat_node_cnt; k++) {
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China l = &lgrp_plat_probe_lat_acct[k];
080575042aba2197b425ebfd52061dea061a9aa1xy * Increment number of occurrences
080575042aba2197b425ebfd52061dea061a9aa1xy * if seen before
080575042aba2197b425ebfd52061dea061a9aa1xy } else if (l->la_value == 0) {
080575042aba2197b425ebfd52061dea061a9aa1xy * Record latency if haven't seen before
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems * Number of latencies should be relative to number of
080575042aba2197b425ebfd52061dea061a9aa1xy * nodes in system:
080575042aba2197b425ebfd52061dea061a9aa1xy * - Same as nodes when nodes <= 2
080575042aba2197b425ebfd52061dea061a9aa1xy * - Less than nodes when nodes > 2
c7770590c6dc06be2588a5c21427e2d823baa989mx * - Greater than 2 when nodes >= 4
3d15c084da89e6f689f1804f3e2e600e5376c4e1chenlu chen - Sun Microsystems - Beijing China (lgrp_plat_node_cnt > 2 &&
080575042aba2197b425ebfd52061dea061a9aa1xy return (-4);
080575042aba2197b425ebfd52061dea061a9aa1xy * There should be more than one occurrence of every latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy * as long as probing is complete
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_probe_nlatencies; i++) {
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng return (-5);
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy return (0);
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy * Set lgroup latencies for 2 level lgroup topology
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyystatic void
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc "MPO only optimizing for local and remote\n");
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (i = 0; i < lgrp_plat_node_cnt; i++) {
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (j = 0; j < lgrp_plat_node_cnt; j++) {
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc if (i == j)
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini * Return time needed to probe from current CPU to memory in given node
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini /* LINTED: set but not used in function */
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini * Determine ID of node containing current CPU
080575042aba2197b425ebfd52061dea061a9aa1xy * Do common work for probing main memory
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems if (lgrp_plat_probe_op == LGRP_PLAT_PROBE_PGCPY) {
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Skip probing any nodes without memory and
080575042aba2197b425ebfd52061dea061a9aa1xy * set probe time to 0
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems if (lgrp_plat_probe_memory[to] == NULL) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Invalidate caches once instead of once every sample
25f2d433de915875c8393f0b0dc14aa155997ad0xy * which should cut cost of probing by a lot
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_flush_cost = gethrtime() - lgrp_plat_flush_cost;
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Probe from current CPU to given memory using specified operation
25f2d433de915875c8393f0b0dc14aa155997ad0xy * and take specified number of samples
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystems * Can't measure probe time if gethrtime() isn't working yet
9b6541b318d01d0d83bfb98699a7f09e35f37951gl return (0);
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * Measure how long it takes to copy page
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * on top of itself
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi buf = lgrp_plat_probe_memory[to] + (i * PAGESIZE);
080575042aba2197b425ebfd52061dea061a9aa1xy * Measure how long it takes to read vendor ID from
return (min);
lgrp_plat_probe(void)
int from;
int to;
for (i = 0; i < lgrp_plat_probe_nrounds; i++)
if (probe_time == 0)
lgrp_plat_main_init(void)
int curnode;
int ht_limit;
if (lgrp_plat_mem_intrlv)
lgrp_plat_probe_time_max == 0)
if (lgrp_plat_probe_memsize == 0)
for (i = 0; i < lgrp_plat_node_cnt; i++) {
int mnode;
lgrp_t *
return (NULL);
return (lgrp);
return (LGRP_DEFAULT_HANDLE);
int mnode;
return (LGRP_DEFAULT_HANDLE);
return (lgrp_topo_initialized ?
static pgcnt_t
switch (query) {
case LGRP_MEM_SIZE_FREE:
case LGRP_MEM_SIZE_AVAIL:
return (npgs);
case LGRP_MEM_SIZE_INSTALL:
return (npgs);
return ((pgcnt_t)0);
int mnode;
switch (query) {
case LGRP_MEM_SIZE_FREE:
case LGRP_MEM_SIZE_AVAIL:
case LGRP_MEM_SIZE_INSTALL:
return (npgs);
return (lgrp_plat_probe_time_max);
lgrp_plat_root_hand(void)
return (LGRP_DEFAULT_HANDLE);