lgrpplat.c revision 2dae3fb5f236a83380b9deea54417c4e1f535121
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * CDDL HEADER START
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * The contents of this file are subject to the terms of the
080575042aba2197b425ebfd52061dea061a9aa1xy * Common Development and Distribution License, Version 1.0 only
080575042aba2197b425ebfd52061dea061a9aa1xy * (the "License"). You may not use this file except in compliance
080575042aba2197b425ebfd52061dea061a9aa1xy * with the License.
080575042aba2197b425ebfd52061dea061a9aa1xy *
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
080575042aba2197b425ebfd52061dea061a9aa1xy * or http://www.opensolaris.org/os/licensing.
080575042aba2197b425ebfd52061dea061a9aa1xy * See the License for the specific language governing permissions
080575042aba2197b425ebfd52061dea061a9aa1xy * and limitations under the License.
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * When distributing Covered Code, include this CDDL HEADER in each
080575042aba2197b425ebfd52061dea061a9aa1xy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
080575042aba2197b425ebfd52061dea061a9aa1xy * If applicable, add the following below this CDDL HEADER, with the
080575042aba2197b425ebfd52061dea061a9aa1xy * fields enclosed by brackets "[]" replaced with your own identifying
080575042aba2197b425ebfd52061dea061a9aa1xy * information: Portions Copyright [yyyy] [name of copyright owner]
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * CDDL HEADER END
080575042aba2197b425ebfd52061dea061a9aa1xy */
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China/*
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * Use is subject to license terms.
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy#pragma ident "%Z%%M% %I% %E% SMI"
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/archsystm.h> /* for {in,out}{b,w,l}() */
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/cmn_err.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/cpupart.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/cpuvar.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/lgrp.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/machsystm.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/memlist.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/memnode.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/mman.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/pci_impl.h> /* for PCI configuration space macros */
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/param.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/promif.h> /* for prom_printf() */
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/systm.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/thread.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/types.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/var.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <sys/x86_archext.h> /* for x86_feature and X86_AMD */
080575042aba2197b425ebfd52061dea061a9aa1xy#include <vm/hat_i86.h>
080575042aba2197b425ebfd52061dea061a9aa1xy#include <vm/seg_kmem.h>
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * lgroup platform support for x86 platforms.
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy#define MAX_NODES 8
080575042aba2197b425ebfd52061dea061a9aa1xy#define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1)
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_CPU_TO_NODE(cpu) (chip_plat_get_chipid(cpu))
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Multiprocessor Opteron machines have Non Uniform Memory Access (NUMA).
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * Until System Affinity Resource Table (SRAT) becomes part of ACPI standard,
080575042aba2197b425ebfd52061dea061a9aa1xy * we need to examine registers in PCI configuration space to determine how
080575042aba2197b425ebfd52061dea061a9aa1xy * many nodes are in the system and which CPUs and memory are in each node.
080575042aba2197b425ebfd52061dea061a9aa1xy * This could be determined by probing all memory from each CPU, but that is
c7770590c6dc06be2588a5c21427e2d823baa989mx * too expensive to do while booting the kernel.
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * NOTE: Using these PCI configuration space registers to determine this
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * locality info is Opteron K8 specific and not guaranteed to work on
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * the next generation Opteron processor. Furthermore, we assume that
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * there is one CPU per node and CPU 0 is in node 0, CPU 1 is in node 1,
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * etc. which should be true for Opteron K8....
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China/*
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * Opteron DRAM Address Map in PCI configuration space gives base and limit
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * of physical memory in each node for Opteron K8. The following constants
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China * and macros define their contents, structure, and access.
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China */
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China/*
080575042aba2197b425ebfd52061dea061a9aa1xy * How many bits to shift Opteron DRAM Address Map base and limit registers
080575042aba2197b425ebfd52061dea061a9aa1xy * to get actual value
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMADDR_LSHIFT_ADDR 8 /* shift left for address */
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMADDR_MASK_OFF 0xFFFFFF /* offset for address */
c7770590c6dc06be2588a5c21427e2d823baa989mx
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems/*
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems * Bit masks defining what's in Opteron DRAM Address Map base register
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China */
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China#define OPT_DRAMBASE_MASK_RE 0x1 /* read enable */
c7770590c6dc06be2588a5c21427e2d823baa989mx#define OPT_DRAMBASE_MASK_WE 0x2 /* write enable */
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China#define OPT_DRAMBASE_MASK_INTRLVEN 0x700 /* interleave */
4d7379630d53d9992780329b674af8c85935e858xiangtao you - Sun Microsystems - Beijing China
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMBASE_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Macros to get values from Opteron DRAM Address Map base register
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy#define OPT_DRAMBASE(reg) \
080575042aba2197b425ebfd52061dea061a9aa1xy (((u_longlong_t)reg & OPT_DRAMBASE_MASK_ADDR) << \
080575042aba2197b425ebfd52061dea061a9aa1xy OPT_DRAMADDR_LSHIFT_ADDR)
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Bit masks defining what's in Opteron DRAM Address Map limit register
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_DSTNODE 0x7 /* destination node */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_INTRLVSEL 0x70 /* interleave select */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT_MASK_ADDR 0xFFFF0000 /* addr bits 39-24 */
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Macros to get values from Opteron DRAM Address Map limit register
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_DRAMLIMIT(reg) \
25f2d433de915875c8393f0b0dc14aa155997ad0xy (((u_longlong_t)reg & OPT_DRAMLIMIT_MASK_ADDR) << \
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China OPT_DRAMADDR_LSHIFT_ADDR)
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Opteron Node ID register in PCI configuration space contains
25f2d433de915875c8393f0b0dc14aa155997ad0xy * number of nodes in system, etc. for Opteron K8. The following
25f2d433de915875c8393f0b0dc14aa155997ad0xy * constants and macros define its contents, structure, and access.
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Bit masks defining what's in Opteron Node ID register
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_NODE_MASK_ID 0x7 /* node ID */
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc#define OPT_NODE_MASK_CNT 0x70 /* node count */
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China#define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * How many bits in Opteron Node ID register to shift right to get actual value
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China */
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China#define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China/*
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Macros to get values from Opteron Node ID register
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_NODE_CNT(reg) \
47b7744cbea59975a6b583125b7ed1ff2ac45313yy ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
47b7744cbea59975a6b583125b7ed1ff2ac45313yy/*
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * PCI configuration space registers accessed by specifying
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * a bus, device, function, and offset. The following constants
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * define the values needed to access Opteron K8 configuration
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * info to determine its node topology
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Opteron PCI configuration space register function values
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy#define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
47b7744cbea59975a6b583125b7ed1ff2ac45313yy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * PCI Configuration Space register offsets
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
c7770590c6dc06be2588a5c21427e2d823baa989mx#define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_PCS_OFF_DRAMBASE 0x40 /* DRAM Base register (node 0) */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron PCI Configuration Space device IDs for nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy#define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */
6ad5fc39c6f3b123ae5588d60fc8dfe068e07bfcsv
c7770590c6dc06be2588a5c21427e2d823baa989mx
3d15c084da89e6f689f1804f3e2e600e5376c4e1chenlu chen - Sun Microsystems - Beijing China/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Bookkeeping for latencies seen during probing (used for verification)
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystemstypedef struct lgrp_plat_latency_acct {
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems hrtime_t la_value; /* latency value */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems int la_count; /* occurrences */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems} lgrp_plat_latency_acct_t;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Choices for probing to determine lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xytypedef enum lgrp_plat_probe_op {
080575042aba2197b425ebfd52061dea061a9aa1xy LGRP_PLAT_PROBE_PGCPY, /* Use page copy */
080575042aba2197b425ebfd52061dea061a9aa1xy LGRP_PLAT_PROBE_VENDOR /* Read vendor ID on Northbridge */
080575042aba2197b425ebfd52061dea061a9aa1xy} lgrp_plat_probe_op_t;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron DRAM address map gives base and limit for physical memory in a node
080575042aba2197b425ebfd52061dea061a9aa1xy */
c7770590c6dc06be2588a5c21427e2d823baa989mxtypedef struct opt_dram_addr_map {
080575042aba2197b425ebfd52061dea061a9aa1xy uint32_t base;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems uint32_t limit;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems} opt_dram_addr_map_t;
080575042aba2197b425ebfd52061dea061a9aa1xy
ae6aa22afeb444ae208c287e7227a4a7c877f17aVenugopal Iyer
ae6aa22afeb444ae208c287e7227a4a7c877f17aVenugopal Iyer/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Starting and ending page for physical memory in node
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xytypedef struct phys_addr_map {
080575042aba2197b425ebfd52061dea061a9aa1xy pfn_t start;
080575042aba2197b425ebfd52061dea061a9aa1xy pfn_t end;
080575042aba2197b425ebfd52061dea061a9aa1xy} phys_addr_map_t;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Opteron DRAM address map for each node
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xystruct opt_dram_addr_map opt_dram_map[MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Node ID register contents for each node
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyuint_t opt_node_info[MAX_NODES];
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Whether memory is interleaved across nodes causing MPO to be disabled
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyint lgrp_plat_mem_intrlv = 0;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Number of nodes in system
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xyuint_t lgrp_plat_node_cnt = 1;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Physical address range for memory in each node
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
4045d94132614e1de2073685a6cdd4fbd86bec33sowminiphys_addr_map_t lgrp_plat_node_memory[MAX_NODES];
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
9b6541b318d01d0d83bfb98699a7f09e35f37951gl * Probe costs (individual and total) and flush cost
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyhrtime_t lgrp_plat_flush_cost = 0;
080575042aba2197b425ebfd52061dea061a9aa1xyhrtime_t lgrp_plat_probe_cost = 0;
080575042aba2197b425ebfd52061dea061a9aa1xyhrtime_t lgrp_plat_probe_cost_total = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Error code for latency adjustment and verification
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyint lgrp_plat_probe_error_code = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * How much latencies were off from minimum values gotten
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyhrtime_t lgrp_plat_probe_errors[MAX_NODES][MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Unique probe latencies and number of occurrences of each
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xylgrp_plat_latency_acct_t lgrp_plat_probe_lat_acct[MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Size of memory buffer in each node for probing
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xysize_t lgrp_plat_probe_memsize = 0;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc * Virtual address of page in each node for probing
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystems */
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystemscaddr_t lgrp_plat_probe_memory[MAX_NODES];
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc/*
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc * Number of unique latencies in probe times
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc */
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611eccint lgrp_plat_probe_nlatencies = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China/*
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China * How many rounds of probing to do
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China */
c7770590c6dc06be2588a5c21427e2d823baa989mxint lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Number of samples to take when probing each node
7941757c1241fe30e30f921910595c8ac6af9ef1xy */
7941757c1241fe30e30f921910595c8ac6af9ef1xyint lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * How to probe to determine lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xylgrp_plat_probe_op_t lgrp_plat_probe_op = LGRP_PLAT_PROBE_VENDOR;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * PFN of page in each node for probing
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xypfn_t lgrp_plat_probe_pfn[MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Whether probe time was suspect (ie. not within tolerance of value that it
080575042aba2197b425ebfd52061dea061a9aa1xy * should match)
ea6b684a18957883cb91b3d22a9d989f986e5a32yy */
ea6b684a18957883cb91b3d22a9d989f986e5a32yyint lgrp_plat_probe_suspect[MAX_NODES][MAX_NODES];
ea6b684a18957883cb91b3d22a9d989f986e5a32yy
ea6b684a18957883cb91b3d22a9d989f986e5a32yy/*
ea6b684a18957883cb91b3d22a9d989f986e5a32yy * How long it takes to access memory from each node
080575042aba2197b425ebfd52061dea061a9aa1xy */
c7770590c6dc06be2588a5c21427e2d823baa989mxhrtime_t lgrp_plat_probe_times[MAX_NODES][MAX_NODES];
c7770590c6dc06be2588a5c21427e2d823baa989mx
c7770590c6dc06be2588a5c21427e2d823baa989mx/*
c7770590c6dc06be2588a5c21427e2d823baa989mx * Min and max node memory probe times seen
c7770590c6dc06be2588a5c21427e2d823baa989mx */
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_time_max = 0;
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_time_min = -1;
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_max[MAX_NODES][MAX_NODES];
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystemshrtime_t lgrp_plat_probe_min[MAX_NODES][MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Allocate lgrp and lgrp stat arrays statically.
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xystatic lgrp_t lgrp_space[NLGRP];
080575042aba2197b425ebfd52061dea061a9aa1xystatic int nlgrps_alloc;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xystruct lgrp_stats lgrp_stats[NLGRP];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy#define CPUID_FAMILY_OPTERON 15
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xyuint_t opt_family = 0;
080575042aba2197b425ebfd52061dea061a9aa1xyuint_t opt_model = 0;
080575042aba2197b425ebfd52061dea061a9aa1xyuint_t opt_probe_func = OPT_PCS_FUNC_DRAM;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Determine whether we're running on an AMD Opteron K8 machine
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyint
080575042aba2197b425ebfd52061dea061a9aa1xyis_opteron(void)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy if (x86_vendor != X86_VENDOR_AMD)
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy if (cpuid_getfamily(CPU) == CPUID_FAMILY_OPTERON)
080575042aba2197b425ebfd52061dea061a9aa1xy return (1);
080575042aba2197b425ebfd52061dea061a9aa1xy else
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy}
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xyint
080575042aba2197b425ebfd52061dea061a9aa1xyplat_lgrphand_to_mem_node(lgrp_handle_t hand)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1)
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy return ((int)hand);
080575042aba2197b425ebfd52061dea061a9aa1xy}
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xylgrp_handle_t
080575042aba2197b425ebfd52061dea061a9aa1xyplat_mem_node_to_lgrphand(int mnode)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1)
080575042aba2197b425ebfd52061dea061a9aa1xy return (LGRP_DEFAULT_HANDLE);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy return ((lgrp_handle_t)mnode);
080575042aba2197b425ebfd52061dea061a9aa1xy}
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xyint
080575042aba2197b425ebfd52061dea061a9aa1xyplat_pfn_to_mem_node(pfn_t pfn)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy int node;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1)
080575042aba2197b425ebfd52061dea061a9aa1xy return (0);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy for (node = 0; node < lgrp_plat_node_cnt; node++) {
080575042aba2197b425ebfd52061dea061a9aa1xy if (pfn >= lgrp_plat_node_memory[node].start &&
080575042aba2197b425ebfd52061dea061a9aa1xy pfn <= lgrp_plat_node_memory[node].end)
080575042aba2197b425ebfd52061dea061a9aa1xy return (node);
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy ASSERT(node < lgrp_plat_node_cnt);
080575042aba2197b425ebfd52061dea061a9aa1xy return (-1);
080575042aba2197b425ebfd52061dea061a9aa1xy}
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Configure memory nodes for machines with more than one node (ie NUMA)
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyvoid
080575042aba2197b425ebfd52061dea061a9aa1xyplat_build_mem_nodes(struct memlist *list)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy pfn_t cur_start, cur_end; /* start & end addr of subrange */
080575042aba2197b425ebfd52061dea061a9aa1xy pfn_t start, end; /* start & end addr of whole range */
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Boot install lists are arranged <addr, len>, ...
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy while (list) {
080575042aba2197b425ebfd52061dea061a9aa1xy int node;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy start = list->address >> PAGESHIFT;
080575042aba2197b425ebfd52061dea061a9aa1xy end = (list->address + list->size - 1) >> PAGESHIFT;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy if (start > physmax) {
080575042aba2197b425ebfd52061dea061a9aa1xy list = list->next;
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy if (end > physmax)
080575042aba2197b425ebfd52061dea061a9aa1xy end = physmax;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * When there is only one memnode, just add memory to memnode
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1) {
080575042aba2197b425ebfd52061dea061a9aa1xy mem_node_add_slice(start, end);
080575042aba2197b425ebfd52061dea061a9aa1xy list = list->next;
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * mem_node_add_slice() expects to get a memory range that
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * is within one memnode, so need to split any memory range
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * that spans multiple memnodes into subranges that are each
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * contained within one memnode when feeding them to
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * mem_node_add_slice()
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng */
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng cur_start = start;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng do {
080575042aba2197b425ebfd52061dea061a9aa1xy node = plat_pfn_to_mem_node(cur_start);
080575042aba2197b425ebfd52061dea061a9aa1xy ASSERT(cur_start >=
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_memory[node].start &&
080575042aba2197b425ebfd52061dea061a9aa1xy cur_start <= lgrp_plat_node_memory[node].end);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy cur_end = end;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * End of current subrange should not span memnodes
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (cur_end > lgrp_plat_node_memory[node].end)
080575042aba2197b425ebfd52061dea061a9aa1xy cur_end = lgrp_plat_node_memory[node].end;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy mem_node_add_slice(cur_start, cur_end);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Next subrange starts after end of current one
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy cur_start = cur_end + 1;
080575042aba2197b425ebfd52061dea061a9aa1xy } while (cur_end < end);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy list = list->next;
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy mem_node_physalign = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy mem_node_pfn_shift = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy}
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Platform-specific initialization of lgroups
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xyvoid
25f2d433de915875c8393f0b0dc14aa155997ad0xylgrp_plat_init(void)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy uint_t bus;
080575042aba2197b425ebfd52061dea061a9aa1xy uint_t dev;
080575042aba2197b425ebfd52061dea061a9aa1xy uint_t node;
080575042aba2197b425ebfd52061dea061a9aa1xy uint_t off;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy extern lgrp_load_t lgrp_expand_proc_thresh;
080575042aba2197b425ebfd52061dea061a9aa1xy extern lgrp_load_t lgrp_expand_proc_diff;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Initialize as a UMA machine if this isn't an Opteron
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (!is_opteron() || lgrp_topo_ht_limit() == 1) {
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_cnt = max_mem_nodes = 1;
080575042aba2197b425ebfd52061dea061a9aa1xy return;
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Read configuration registers from PCI configuration space to
080575042aba2197b425ebfd52061dea061a9aa1xy * determine node information, which memory is in each node, etc.
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * Write to PCI configuration space address register to specify
080575042aba2197b425ebfd52061dea061a9aa1xy * which configuration register to read and read/write PCI
ea6b684a18957883cb91b3d22a9d989f986e5a32yy * configuration space data register to get/set contents
ea6b684a18957883cb91b3d22a9d989f986e5a32yy */
ea6b684a18957883cb91b3d22a9d989f986e5a32yy bus = OPT_PCS_BUS_CONFIG;
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy dev = OPT_PCS_DEV_NODE0;
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy off = OPT_PCS_OFF_DRAMBASE;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems /*
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Read node ID register for node 0 to get node count
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy */
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy outl(PCI_CONFADD, PCI_CADDR1(bus, dev, OPT_PCS_FUNC_HT,
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy OPT_PCS_OFF_NODEID));
080575042aba2197b425ebfd52061dea061a9aa1xy opt_node_info[0] = inl(PCI_CONFDATA);
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_cnt = OPT_NODE_CNT(opt_node_info[0]) + 1;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy for (node = 0; node < lgrp_plat_node_cnt; node++) {
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Read node ID register (except for node 0 which we just read)
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (node > 0) {
080575042aba2197b425ebfd52061dea061a9aa1xy outl(PCI_CONFADD, PCI_CADDR1(bus, dev,
080575042aba2197b425ebfd52061dea061a9aa1xy OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID));
080575042aba2197b425ebfd52061dea061a9aa1xy opt_node_info[node] = inl(PCI_CONFDATA);
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Read DRAM base and limit registers which specify
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China * physical memory range of each node
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China */
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China outl(PCI_CONFADD, PCI_CADDR1(bus, dev, OPT_PCS_FUNC_ADDRMAP,
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China off));
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China opt_dram_map[node].base = inl(PCI_CONFDATA);
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China if (opt_dram_map[node].base & OPT_DRAMBASE_MASK_INTRLVEN)
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China lgrp_plat_mem_intrlv++;
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China off += 4; /* limit register offset */
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China outl(PCI_CONFADD, PCI_CADDR1(bus, dev, OPT_PCS_FUNC_ADDRMAP,
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China off));
3f64cd552fee350c8075ec62765e9a6f9caef1a8guoqing zhu - Sun Microsystems - Beijing China opt_dram_map[node].limit = inl(PCI_CONFDATA);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Increment device number to next node and register offset for
25f2d433de915875c8393f0b0dc14aa155997ad0xy * DRAM base register of next node
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy off += 4;
25f2d433de915875c8393f0b0dc14aa155997ad0xy dev++;
080575042aba2197b425ebfd52061dea061a9aa1xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Get PFN for first page in each node,
25f2d433de915875c8393f0b0dc14aa155997ad0xy * so we can probe memory to determine latency topology
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_pfn[node] =
080575042aba2197b425ebfd52061dea061a9aa1xy btop(OPT_DRAMBASE(opt_dram_map[node].base));
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Remember physical address range of each node for use later
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_memory[node].start =
080575042aba2197b425ebfd52061dea061a9aa1xy btop(OPT_DRAMBASE(opt_dram_map[node].base));
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_memory[node].end =
080575042aba2197b425ebfd52061dea061a9aa1xy btop(OPT_DRAMLIMIT(opt_dram_map[node].limit) |
080575042aba2197b425ebfd52061dea061a9aa1xy OPT_DRAMADDR_MASK_OFF);
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Only use one memory node if memory is interleaved between any nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (lgrp_plat_mem_intrlv) {
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_node_cnt = max_mem_nodes = 1;
080575042aba2197b425ebfd52061dea061a9aa1xy (void) lgrp_topo_ht_limit_set(1);
080575042aba2197b425ebfd52061dea061a9aa1xy } else {
080575042aba2197b425ebfd52061dea061a9aa1xy max_mem_nodes = lgrp_plat_node_cnt;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Probing errors can mess up the lgroup topology and force us
25f2d433de915875c8393f0b0dc14aa155997ad0xy * fall back to a 2 level lgroup topology. Here we bound how
25f2d433de915875c8393f0b0dc14aa155997ad0xy * tall the lgroup topology can grow in hopes of avoiding any
080575042aba2197b425ebfd52061dea061a9aa1xy * anamolies in probing from messing up the lgroup topology
080575042aba2197b425ebfd52061dea061a9aa1xy * by limiting the accuracy of the latency topology.
25f2d433de915875c8393f0b0dc14aa155997ad0xy *
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Assume that nodes will at least be configured in a ring,
080575042aba2197b425ebfd52061dea061a9aa1xy * so limit height of lgroup topology to be less than number
080575042aba2197b425ebfd52061dea061a9aa1xy * of nodes on a system with 4 or more nodes
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (lgrp_plat_node_cnt >= 4 &&
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_topo_ht_limit() == lgrp_topo_ht_limit_default())
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Lgroups on Opteron architectures have but a single physical
080575042aba2197b425ebfd52061dea061a9aa1xy * processor. Tune lgrp_expand_proc_thresh and lgrp_expand_proc_diff
080575042aba2197b425ebfd52061dea061a9aa1xy * so that lgrp_choose() will spread things out aggressively.
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_expand_proc_diff = 0;
ea6b684a18957883cb91b3d22a9d989f986e5a32yy}
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy
ea6b684a18957883cb91b3d22a9d989f986e5a32yy/*
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to
080575042aba2197b425ebfd52061dea061a9aa1xy * be considered same
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems */
080575042aba2197b425ebfd52061dea061a9aa1xy#define LGRP_LAT_TOLERANCE_SHIFT 4
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xyint lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
080575042aba2197b425ebfd52061dea061a9aa1xy * Adjust latencies between nodes to be symmetric, normalize latencies between
080575042aba2197b425ebfd52061dea061a9aa1xy * any nodes that are within some tolerance to be same, and make local
25f2d433de915875c8393f0b0dc14aa155997ad0xy * latencies be same
080575042aba2197b425ebfd52061dea061a9aa1xy */
c7770590c6dc06be2588a5c21427e2d823baa989mxstatic void
7941757c1241fe30e30f921910595c8ac6af9ef1xylgrp_plat_latency_adjust(void)
7941757c1241fe30e30f921910595c8ac6af9ef1xy{
7941757c1241fe30e30f921910595c8ac6af9ef1xy int i;
7941757c1241fe30e30f921910595c8ac6af9ef1xy int j;
c7770590c6dc06be2588a5c21427e2d823baa989mx int k;
c7770590c6dc06be2588a5c21427e2d823baa989mx int l;
c7770590c6dc06be2588a5c21427e2d823baa989mx u_longlong_t max;
c7770590c6dc06be2588a5c21427e2d823baa989mx u_longlong_t min;
c7770590c6dc06be2588a5c21427e2d823baa989mx u_longlong_t t;
7941757c1241fe30e30f921910595c8ac6af9ef1xy u_longlong_t t1;
080575042aba2197b425ebfd52061dea061a9aa1xy u_longlong_t t2;
080575042aba2197b425ebfd52061dea061a9aa1xy const lgrp_config_flag_t cflag = LGRP_CONFIG_LATENCY_CHANGE;
080575042aba2197b425ebfd52061dea061a9aa1xy int lat_corrected[MAX_NODES][MAX_NODES];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Nothing to do when this is an UMA machine
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1)
25f2d433de915875c8393f0b0dc14aa155997ad0xy return;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Make sure that latencies are symmetric between any two nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy * (ie. latency(node0, node1) == latency(node1, node0))
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
080575042aba2197b425ebfd52061dea061a9aa1xy t1 = lgrp_plat_probe_times[i][j];
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystems t2 = lgrp_plat_probe_times[j][i];
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy if (t1 == 0 || t2 == 0 || t1 == t2)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Latencies should be same
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Use minimum of two latencies which should be same
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Track suspect probe times not within tolerance of
25f2d433de915875c8393f0b0dc14aa155997ad0xy * min value
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Remember how much values are corrected by
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1 > t2) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = t2;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_errors[i][j] += t1 - t2;
080575042aba2197b425ebfd52061dea061a9aa1xy if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) {
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_suspect[i][j]++;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_suspect[j][i]++;
080575042aba2197b425ebfd52061dea061a9aa1xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy } else if (t2 > t1) {
080575042aba2197b425ebfd52061dea061a9aa1xy t = t1;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_errors[j][i] += t2 - t1;
080575042aba2197b425ebfd52061dea061a9aa1xy if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) {
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_suspect[i][j]++;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_suspect[j][i]++;
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_times[i][j] =
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_times[j][i] = t;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_config(cflag, t1, t);
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_config(cflag, t2, t);
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Keep track of which latencies get corrected
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < MAX_NODES; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < MAX_NODES; j++)
080575042aba2197b425ebfd52061dea061a9aa1xy lat_corrected[i][j] = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * For every two nodes, see whether there is another pair of nodes which
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * are about the same distance apart and make the latencies be the same
080575042aba2197b425ebfd52061dea061a9aa1xy * if they are close enough together
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++)
080575042aba2197b425ebfd52061dea061a9aa1xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Pick one pair of nodes (i, j)
080575042aba2197b425ebfd52061dea061a9aa1xy * and get latency between them
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
080575042aba2197b425ebfd52061dea061a9aa1xy t1 = lgrp_plat_probe_times[i][j];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip this pair of nodes if there isn't a latency
080575042aba2197b425ebfd52061dea061a9aa1xy * for it yet
080575042aba2197b425ebfd52061dea061a9aa1xy */
47b7744cbea59975a6b583125b7ed1ff2ac45313yy if (t1 == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy for (k = 0; k < lgrp_plat_node_cnt; k++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (l = 0; l < lgrp_plat_node_cnt; l++) {
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Pick another pair of nodes (k, l)
080575042aba2197b425ebfd52061dea061a9aa1xy * not same as (i, j) and get latency
080575042aba2197b425ebfd52061dea061a9aa1xy * between them
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (k == i && l == j)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy t2 = lgrp_plat_probe_times[k][l];
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip this pair of nodes if there
080575042aba2197b425ebfd52061dea061a9aa1xy * isn't a latency for it yet
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy if (t2 == 0)
25f2d433de915875c8393f0b0dc14aa155997ad0xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Skip nodes (k, l) if they already
25f2d433de915875c8393f0b0dc14aa155997ad0xy * have same latency as (i, j) or
080575042aba2197b425ebfd52061dea061a9aa1xy * their latency isn't close enough to
080575042aba2197b425ebfd52061dea061a9aa1xy * be considered/made the same
080575042aba2197b425ebfd52061dea061a9aa1xy */
c7770590c6dc06be2588a5c21427e2d823baa989mx if (t1 == t2 || (t1 > t2 && t1 - t2 >
080575042aba2197b425ebfd52061dea061a9aa1xy t1 >> lgrp_plat_probe_lt_shift) ||
c7770590c6dc06be2588a5c21427e2d823baa989mx (t2 > t1 && t2 - t1 >
080575042aba2197b425ebfd52061dea061a9aa1xy t2 >> lgrp_plat_probe_lt_shift))
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Make latency(i, j) same as
25f2d433de915875c8393f0b0dc14aa155997ad0xy * latency(k, l), try to use latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy * that has been adjusted already to get
25f2d433de915875c8393f0b0dc14aa155997ad0xy * more consistency (if possible), and
25f2d433de915875c8393f0b0dc14aa155997ad0xy * remember which latencies were
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * adjusted for next time
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (lat_corrected[i][j]) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = t1;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_config(cflag, t2, t);
25f2d433de915875c8393f0b0dc14aa155997ad0xy t2 = t;
25f2d433de915875c8393f0b0dc14aa155997ad0xy } else if (lat_corrected[k][l]) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = t2;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_config(cflag, t1, t);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng t1 = t;
25f2d433de915875c8393f0b0dc14aa155997ad0xy } else {
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1 > t2)
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = t2;
25f2d433de915875c8393f0b0dc14aa155997ad0xy else
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = t1;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_config(cflag, t1, t);
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_config(cflag, t2, t);
25f2d433de915875c8393f0b0dc14aa155997ad0xy t1 = t2 = t;
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_times[i][j] =
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_times[k][l] = t;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy lat_corrected[i][j] =
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems lat_corrected[k][l] = 1;
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Local latencies should be same
080575042aba2197b425ebfd52061dea061a9aa1xy * - Find min and max local latencies
080575042aba2197b425ebfd52061dea061a9aa1xy * - Make all local latencies be minimum
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems */
080575042aba2197b425ebfd52061dea061a9aa1xy min = -1;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng max = 0;
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++) {
080575042aba2197b425ebfd52061dea061a9aa1xy t = lgrp_plat_probe_times[i][i];
080575042aba2197b425ebfd52061dea061a9aa1xy if (t == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
080575042aba2197b425ebfd52061dea061a9aa1xy if (min == -1 || t < min)
080575042aba2197b425ebfd52061dea061a9aa1xy min = t;
080575042aba2197b425ebfd52061dea061a9aa1xy if (t > max)
080575042aba2197b425ebfd52061dea061a9aa1xy max = t;
080575042aba2197b425ebfd52061dea061a9aa1xy }
080575042aba2197b425ebfd52061dea061a9aa1xy if (min != max) {
080575042aba2197b425ebfd52061dea061a9aa1xy for (i = 0; i < lgrp_plat_node_cnt; i++) {
080575042aba2197b425ebfd52061dea061a9aa1xy int local;
080575042aba2197b425ebfd52061dea061a9aa1xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy local = lgrp_plat_probe_times[i][i];
080575042aba2197b425ebfd52061dea061a9aa1xy if (local == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Track suspect probe times that aren't within
25f2d433de915875c8393f0b0dc14aa155997ad0xy * tolerance of minimum local latency and how much
25f2d433de915875c8393f0b0dc14aa155997ad0xy * probe times are corrected by
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng */
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (local - min > min >> lgrp_plat_probe_lt_shift)
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_suspect[i][i]++;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems lgrp_plat_probe_errors[i][i] += local - min;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems /*
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Make local latencies be minimum
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems lgrp_config(cflag, local, min);
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems lgrp_plat_probe_times[i][i] = min;
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems }
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems }
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng /*
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * Determine max probe time again since just adjusted latencies
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng */
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng lgrp_plat_probe_time_max = 0;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng for (i = 0; i < lgrp_plat_node_cnt; i++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy t = lgrp_plat_probe_times[i][j];
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t > lgrp_plat_probe_time_max)
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_time_max = t;
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy}
46ebaa55cce1df60528a191312d12199d38a4493Miles Xu, Sun Microsystems
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China
25f2d433de915875c8393f0b0dc14aa155997ad0xy/*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Verify following about latencies between nodes:
25f2d433de915875c8393f0b0dc14aa155997ad0xy *
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a))
25f2d433de915875c8393f0b0dc14aa155997ad0xy * - Local latencies same
080575042aba2197b425ebfd52061dea061a9aa1xy * - Local < remote
080575042aba2197b425ebfd52061dea061a9aa1xy * - Number of latencies seen is reasonable
080575042aba2197b425ebfd52061dea061a9aa1xy * - Number of occurrences of a given latency should be more than 1
080575042aba2197b425ebfd52061dea061a9aa1xy *
080575042aba2197b425ebfd52061dea061a9aa1xy * Returns:
080575042aba2197b425ebfd52061dea061a9aa1xy * 0 Success
080575042aba2197b425ebfd52061dea061a9aa1xy * -1 Not symmetric
25f2d433de915875c8393f0b0dc14aa155997ad0xy * -2 Local latencies not same
080575042aba2197b425ebfd52061dea061a9aa1xy * -3 Local >= remote
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy * -4 Wrong number of latencies
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * -5 Not enough occurrences of given latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xystatic int
25f2d433de915875c8393f0b0dc14aa155997ad0xylgrp_plat_latency_verify(void)
080575042aba2197b425ebfd52061dea061a9aa1xy{
080575042aba2197b425ebfd52061dea061a9aa1xy int i;
d5c3073dbbd835e1e9b7dca0c6c770cf3cc20afachenlu chen - Sun Microsystems - Beijing China int j;
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_latency_acct_t *l;
25f2d433de915875c8393f0b0dc14aa155997ad0xy int probed;
25f2d433de915875c8393f0b0dc14aa155997ad0xy u_longlong_t t1;
a2e9a8308e6b9832ce4d7b848660483fc31d1dc7cc u_longlong_t t2;
7941757c1241fe30e30f921910595c8ac6af9ef1xy
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Nothing to do when this is an UMA machine, lgroup topology is
25f2d433de915875c8393f0b0dc14aa155997ad0xy * limited to 2 levels, or there aren't any probe times yet
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (max_mem_nodes == 1 || lgrp_topo_levels < 2 ||
25f2d433de915875c8393f0b0dc14aa155997ad0xy (lgrp_plat_probe_time_max == 0 && lgrp_plat_probe_time_min == -1))
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems return (0);
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Make sure that latencies are symmetric between any two nodes
25f2d433de915875c8393f0b0dc14aa155997ad0xy * (ie. latency(node0, node1) == latency(node1, node0))
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems */
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner for (j = 0; j < lgrp_plat_node_cnt; j++) {
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner t1 = lgrp_plat_probe_times[i][j];
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner t2 = lgrp_plat_probe_times[j][i];
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner if (t1 == 0 || t2 == 0 || t1 == t2)
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner continue;
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner return (-1);
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner }
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner /*
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner * Local latencies should be same
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner */
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner t1 = lgrp_plat_probe_times[0][0];
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner for (i = 1; i < lgrp_plat_node_cnt; i++) {
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner t2 = lgrp_plat_probe_times[i][i];
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner if (t2 == 0)
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner continue;
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner if (t1 == 0) {
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner t1 = t2;
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner continue;
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner }
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner if (t1 != t2)
29fd2c1627adafbe0a83726c224e60b5514c7736David Höppner return (-2);
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Local latencies should be less than remote
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China */
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_node_cnt; i++)
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (j = 0; j < lgrp_plat_node_cnt; j++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy t2 = lgrp_plat_probe_times[i][j];
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (i == j || t2 == 0)
25f2d433de915875c8393f0b0dc14aa155997ad0xy continue;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1 >= t2)
ede5269ebe7fa3787cc9b58c3781b639c578f93dchenlu chen - Sun Microsystems - Beijing China return (-3);
25f2d433de915875c8393f0b0dc14aa155997ad0xy }
080575042aba2197b425ebfd52061dea061a9aa1xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Rest of checks are not very useful for machines with less than
25f2d433de915875c8393f0b0dc14aa155997ad0xy * 4 nodes (which means less than 3 latencies on Opteron)
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy if (lgrp_plat_node_cnt < 4)
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China /*
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China * Need to see whether done probing in order to verify number of
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China * latencies are correct
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China */
1bc1c72171ee0c5d77203ddffb90219d991bd6e8guoqing zhu - Sun Microsystems - Beijing China probed = 0;
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (i = 0; i < lgrp_plat_node_cnt; i++)
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystems if (lgrp_plat_probe_times[i][i])
592a4d85662412bade15f3d9e9e0cbcf8514348ccc probed++;
592a4d85662412bade15f3d9e9e0cbcf8514348ccc
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc if (probed != lgrp_plat_node_cnt)
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
25f2d433de915875c8393f0b0dc14aa155997ad0xy
25f2d433de915875c8393f0b0dc14aa155997ad0xy /*
7941757c1241fe30e30f921910595c8ac6af9ef1xy * Determine number of unique latencies seen in probe times,
25f2d433de915875c8393f0b0dc14aa155997ad0xy * their values, and number of occurrences of each
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_nlatencies = 0;
25f2d433de915875c8393f0b0dc14aa155997ad0xy bzero(lgrp_plat_probe_lat_acct,
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng MAX_NODES * sizeof (lgrp_plat_latency_acct_t));
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng for (i = 0; i < lgrp_plat_node_cnt; i++) {
47b7744cbea59975a6b583125b7ed1ff2ac45313yy for (j = 0; j < lgrp_plat_node_cnt; j++) {
47b7744cbea59975a6b583125b7ed1ff2ac45313yy int k;
47b7744cbea59975a6b583125b7ed1ff2ac45313yy
47b7744cbea59975a6b583125b7ed1ff2ac45313yy /*
47b7744cbea59975a6b583125b7ed1ff2ac45313yy * Look at each probe time
47b7744cbea59975a6b583125b7ed1ff2ac45313yy */
080575042aba2197b425ebfd52061dea061a9aa1xy t1 = lgrp_plat_probe_times[i][j];
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (t1 == 0)
080575042aba2197b425ebfd52061dea061a9aa1xy continue;
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Account for unique latencies
080575042aba2197b425ebfd52061dea061a9aa1xy */
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China for (k = 0; k < lgrp_plat_node_cnt; k++) {
96ea4e937aa9d4d9bcb316497c698a4f411380b7changqing li - Sun Microsystems - Beijing China l = &lgrp_plat_probe_lat_acct[k];
080575042aba2197b425ebfd52061dea061a9aa1xy if (t1 == l->la_value) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Increment number of occurrences
080575042aba2197b425ebfd52061dea061a9aa1xy * if seen before
080575042aba2197b425ebfd52061dea061a9aa1xy */
080575042aba2197b425ebfd52061dea061a9aa1xy l->la_count++;
080575042aba2197b425ebfd52061dea061a9aa1xy break;
080575042aba2197b425ebfd52061dea061a9aa1xy } else if (l->la_value == 0) {
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Record latency if haven't seen before
9b6541b318d01d0d83bfb98699a7f09e35f37951gl */
080575042aba2197b425ebfd52061dea061a9aa1xy l->la_value = t1;
080575042aba2197b425ebfd52061dea061a9aa1xy l->la_count++;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_nlatencies++;
080575042aba2197b425ebfd52061dea061a9aa1xy break;
080575042aba2197b425ebfd52061dea061a9aa1xy }
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China }
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China }
0c56b8d9d1c30af2e78d0a6fd8a6a70aa5310099changqing li - Sun Microsystems - Beijing China }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
43a176879122f7764767ab85eb539f89d65cdd67Miles Xu, Sun Microsystems * Number of latencies should be relative to number of
080575042aba2197b425ebfd52061dea061a9aa1xy * nodes in system:
080575042aba2197b425ebfd52061dea061a9aa1xy * - Same as nodes when nodes <= 2
080575042aba2197b425ebfd52061dea061a9aa1xy * - Less than nodes when nodes > 2
c7770590c6dc06be2588a5c21427e2d823baa989mx * - Greater than 2 when nodes >= 4
c7770590c6dc06be2588a5c21427e2d823baa989mx */
c7770590c6dc06be2588a5c21427e2d823baa989mx if ((lgrp_plat_node_cnt <= 2 &&
c7770590c6dc06be2588a5c21427e2d823baa989mx lgrp_plat_probe_nlatencies != lgrp_plat_node_cnt) ||
3d15c084da89e6f689f1804f3e2e600e5376c4e1chenlu chen - Sun Microsystems - Beijing China (lgrp_plat_node_cnt > 2 &&
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_nlatencies >= lgrp_plat_node_cnt) ||
080575042aba2197b425ebfd52061dea061a9aa1xy (lgrp_plat_node_cnt >= 4 && lgrp_topo_levels >= 3 &&
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_nlatencies <= 2))
080575042aba2197b425ebfd52061dea061a9aa1xy return (-4);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * There should be more than one occurrence of every latency
25f2d433de915875c8393f0b0dc14aa155997ad0xy * as long as probing is complete
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_probe_nlatencies; i++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy l = &lgrp_plat_probe_lat_acct[i];
25f2d433de915875c8393f0b0dc14aa155997ad0xy if (l->la_count <= 1)
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng return (-5);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng }
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy return (0);
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy}
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy/*
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy * Set lgroup latencies for 2 level lgroup topology
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyy */
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyystatic void
4914a7d0d1ee59f8cc21b19bfd7979cb65681eacyylgrp_plat_2level_setup(void)
9b6541b318d01d0d83bfb98699a7f09e35f37951gl{
9b6541b318d01d0d83bfb98699a7f09e35f37951gl int i;
9b6541b318d01d0d83bfb98699a7f09e35f37951gl
9b6541b318d01d0d83bfb98699a7f09e35f37951gl if (lgrp_plat_node_cnt >= 4)
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc cmn_err(CE_NOTE,
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc "MPO only optimizing for local and remote\n");
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (i = 0; i < lgrp_plat_node_cnt; i++) {
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc int j;
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc for (j = 0; j < lgrp_plat_node_cnt; j++) {
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc if (i == j)
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini lgrp_plat_probe_times[i][j] = 2;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini else
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini lgrp_plat_probe_times[i][j] = 3;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini }
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini }
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini lgrp_plat_probe_time_min = 2;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini lgrp_plat_probe_time_max = 3;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini}
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini/*
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini * Return time needed to probe from current CPU to memory in given node
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini */
4045d94132614e1de2073685a6cdd4fbd86bec33sowministatic hrtime_t
4045d94132614e1de2073685a6cdd4fbd86bec33sowminilgrp_plat_probe_time(int to)
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini{
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini caddr_t buf;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini uint_t dev;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini /* LINTED: set but not used in function */
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini volatile uint_t dev_vendor;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini hrtime_t elapsed;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini hrtime_t end;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini int from;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini int i;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini int ipl;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini hrtime_t max;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini hrtime_t min;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini hrtime_t start;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini extern int use_sse_pagecopy;
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini /*
4045d94132614e1de2073685a6cdd4fbd86bec33sowmini * Determine ID of node containing current CPU
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy from = LGRP_PLAT_CPU_TO_NODE(CPU);
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Do common work for probing main memory
080575042aba2197b425ebfd52061dea061a9aa1xy */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems if (lgrp_plat_probe_op == LGRP_PLAT_PROBE_PGCPY) {
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems /*
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Skip probing any nodes without memory and
080575042aba2197b425ebfd52061dea061a9aa1xy * set probe time to 0
080575042aba2197b425ebfd52061dea061a9aa1xy */
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems if (lgrp_plat_probe_memory[to] == NULL) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_times[from][to] = 0;
25f2d433de915875c8393f0b0dc14aa155997ad0xy return (0);
3fb4efef75bbb3a06a68d5ff59c33df03c73f6c9changqing li - Sun Microsystems - Beijing China }
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Invalidate caches once instead of once every sample
25f2d433de915875c8393f0b0dc14aa155997ad0xy * which should cut cost of probing by a lot
25f2d433de915875c8393f0b0dc14aa155997ad0xy */
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_flush_cost = gethrtime();
ae6aa22afeb444ae208c287e7227a4a7c877f17aVenugopal Iyer invalidate_cache();
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_flush_cost = gethrtime() - lgrp_plat_flush_cost;
080575042aba2197b425ebfd52061dea061a9aa1xy lgrp_plat_probe_cost_total += lgrp_plat_flush_cost;
080575042aba2197b425ebfd52061dea061a9aa1xy }
25f2d433de915875c8393f0b0dc14aa155997ad0xy
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer /*
25f2d433de915875c8393f0b0dc14aa155997ad0xy * Probe from current CPU to given memory using specified operation
25f2d433de915875c8393f0b0dc14aa155997ad0xy * and take specified number of samples
080575042aba2197b425ebfd52061dea061a9aa1xy */
25f2d433de915875c8393f0b0dc14aa155997ad0xy max = 0;
25f2d433de915875c8393f0b0dc14aa155997ad0xy min = -1;
25f2d433de915875c8393f0b0dc14aa155997ad0xy for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
25f2d433de915875c8393f0b0dc14aa155997ad0xy lgrp_plat_probe_cost = gethrtime();
25f2d433de915875c8393f0b0dc14aa155997ad0xy
080575042aba2197b425ebfd52061dea061a9aa1xy /*
caf05df5c10c960028f122b1b02a3f7d8f892c31Miles Xu, Sun Microsystems * Can't measure probe time if gethrtime() isn't working yet
080575042aba2197b425ebfd52061dea061a9aa1xy */
9b6541b318d01d0d83bfb98699a7f09e35f37951gl if (lgrp_plat_probe_cost == 0 && gethrtime() == 0)
9b6541b318d01d0d83bfb98699a7f09e35f37951gl return (0);
9b6541b318d01d0d83bfb98699a7f09e35f37951gl
837c1ac4e72b7d86278cca88b1075af557f7d161Stephen Hanson switch (lgrp_plat_probe_op) {
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc
9ce7e93c0e8e6d2b400f40e9c5742b1d6682611ecc case LGRP_PLAT_PROBE_PGCPY:
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi default:
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi /*
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * Measure how long it takes to copy page
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi * on top of itself
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi */
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi buf = lgrp_plat_probe_memory[to] + (i * PAGESIZE);
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi kpreempt_disable();
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi ipl = splhigh();
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi start = gethrtime();
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi if (use_sse_pagecopy)
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi hwblkpagecopy(buf, buf);
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi else
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi bcopy(buf, buf, PAGESIZE);
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi end = gethrtime();
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi elapsed = end - start;
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi splx(ipl);
49b7860084dbba18bc00b29413d6182197f9fe93Robert Mustacchi kpreempt_enable();
080575042aba2197b425ebfd52061dea061a9aa1xy break;
080575042aba2197b425ebfd52061dea061a9aa1xy
080575042aba2197b425ebfd52061dea061a9aa1xy case LGRP_PLAT_PROBE_VENDOR:
0f70fbf80d71251e7928b3122fb4848c2f92a5c6xy /*
080575042aba2197b425ebfd52061dea061a9aa1xy * Measure how long it takes to read vendor ID from
54e0d7a5e8285a3f01a0db8db1246ac7cac94d81Miles Xu, Sun Microsystems * Northbridge
ea6b684a18957883cb91b3d22a9d989f986e5a32yy */
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng dev = OPT_PCS_DEV_NODE0 + to;
080575042aba2197b425ebfd52061dea061a9aa1xy kpreempt_disable();
080575042aba2197b425ebfd52061dea061a9aa1xy ipl = spl8();
080575042aba2197b425ebfd52061dea061a9aa1xy outl(PCI_CONFADD, PCI_CADDR1(0, dev, opt_probe_func,
080575042aba2197b425ebfd52061dea061a9aa1xy OPT_PCS_OFF_VENDOR));
080575042aba2197b425ebfd52061dea061a9aa1xy start = gethrtime();
080575042aba2197b425ebfd52061dea061a9aa1xy dev_vendor = inl(PCI_CONFDATA);
end = gethrtime();
elapsed = end - start;
splx(ipl);
kpreempt_enable();
break;
}
lgrp_plat_probe_cost = gethrtime() - lgrp_plat_probe_cost;
lgrp_plat_probe_cost_total += lgrp_plat_probe_cost;
if (min == -1 || elapsed < min)
min = elapsed;
if (elapsed > max)
max = elapsed;
}
/*
* Update minimum and maximum probe times between
* these two nodes
*/
if (min < lgrp_plat_probe_min[from][to] ||
lgrp_plat_probe_min[from][to] == 0)
lgrp_plat_probe_min[from][to] = min;
if (max > lgrp_plat_probe_max[from][to])
lgrp_plat_probe_max[from][to] = max;
return (min);
}
/*
* Probe memory in each node from current CPU to determine latency topology
*/
void
lgrp_plat_probe(void)
{
int from;
int i;
hrtime_t probe_time;
int to;
if (max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
return;
/*
* Determine ID of node containing current CPU
*/
from = LGRP_PLAT_CPU_TO_NODE(CPU);
/*
* Don't need to probe if got times already
*/
if (lgrp_plat_probe_times[from][from] != 0)
return;
/*
* Read vendor ID in Northbridge or read and write page(s)
* in each node from current CPU and remember how long it takes,
* so we can build latency topology of machine later.
* This should approximate the memory latency between each node.
*/
for (i = 0; i < lgrp_plat_probe_nrounds; i++)
for (to = 0; to < lgrp_plat_node_cnt; to++) {
/*
* Get probe time and bail out if can't get it yet
*/
probe_time = lgrp_plat_probe_time(to);
if (probe_time == 0)
return;
/*
* Keep lowest probe time as latency between nodes
*/
if (lgrp_plat_probe_times[from][to] == 0 ||
probe_time < lgrp_plat_probe_times[from][to])
lgrp_plat_probe_times[from][to] = probe_time;
/*
* Update overall minimum and maximum probe times
* across all nodes
*/
if (probe_time < lgrp_plat_probe_time_min ||
lgrp_plat_probe_time_min == -1)
lgrp_plat_probe_time_min = probe_time;
if (probe_time > lgrp_plat_probe_time_max)
lgrp_plat_probe_time_max = probe_time;
}
/*
* - Fix up latencies such that local latencies are same,
* latency(i, j) == latency(j, i), etc. (if possible)
*
* - Verify that latencies look ok
*
* - Fallback to just optimizing for local and remote if
* latencies didn't look right
*/
lgrp_plat_latency_adjust();
lgrp_plat_probe_error_code = lgrp_plat_latency_verify();
if (lgrp_plat_probe_error_code)
lgrp_plat_2level_setup();
}
/*
* Platform-specific initialization
*/
void
lgrp_plat_main_init(void)
{
int curnode;
int ht_limit;
int i;
/*
* Print a notice that MPO is disabled when memory is interleaved
* across nodes....Would do this when it is discovered, but can't
* because it happens way too early during boot....
*/
if (lgrp_plat_mem_intrlv)
cmn_err(CE_NOTE,
"MPO disabled because memory is interleaved\n");
/*
* Don't bother to do any probing if there is only one node or the
* height of the lgroup topology less than or equal to 2
*/
ht_limit = lgrp_topo_ht_limit();
if (max_mem_nodes == 1 || ht_limit <= 2) {
/*
* Setup lgroup latencies for 2 level lgroup topology
* (ie. local and remote only) if they haven't been set yet
*/
if (ht_limit == 2 && lgrp_plat_probe_time_min == -1 &&
lgrp_plat_probe_time_max == 0)
lgrp_plat_2level_setup();
return;
}
if (lgrp_plat_probe_op == LGRP_PLAT_PROBE_VENDOR) {
/*
* Should have been able to probe from CPU 0 when it was added
* to lgroup hierarchy, but may not have been able to then
* because it happens so early in boot that gethrtime() hasn't
* been initialized. (:-(
*/
curnode = LGRP_PLAT_CPU_TO_NODE(CPU);
if (lgrp_plat_probe_times[curnode][curnode] == 0)
lgrp_plat_probe();
return;
}
/*
* When probing memory, use one page for every sample to determine
* lgroup topology and taking multiple samples
*/
if (lgrp_plat_probe_memsize == 0)
lgrp_plat_probe_memsize = PAGESIZE *
lgrp_plat_probe_nsamples;
/*
* Map memory in each node needed for probing to determine latency
* topology
*/
for (i = 0; i < lgrp_plat_node_cnt; i++) {
int mnode;
/*
* Skip this node and leave its probe page NULL
* if it doesn't have any memory
*/
mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i);
if (!mem_node_config[mnode].exists) {
lgrp_plat_probe_memory[i] = NULL;
continue;
}
/*
* Allocate one kernel virtual page
*/
lgrp_plat_probe_memory[i] = vmem_alloc(heap_arena,
lgrp_plat_probe_memsize, VM_NOSLEEP);
if (lgrp_plat_probe_memory[i] == NULL) {
cmn_err(CE_WARN,
"lgrp_plat_main_init: couldn't allocate memory");
return;
}
/*
* Map virtual page to first page in node
*/
hat_devload(kas.a_hat, lgrp_plat_probe_memory[i],
lgrp_plat_probe_memsize,
lgrp_plat_probe_pfn[i],
PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
HAT_LOAD_NOCONSIST);
}
/*
* Probe from current CPU
*/
lgrp_plat_probe();
}
/*
* Allocate additional space for an lgroup.
*/
/* ARGSUSED */
lgrp_t *
lgrp_plat_alloc(lgrp_id_t lgrpid)
{
lgrp_t *lgrp;
lgrp = &lgrp_space[nlgrps_alloc++];
if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
return (NULL);
return (lgrp);
}
/*
* Platform handling for (re)configuration changes
*/
/* ARGSUSED */
void
lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
{
}
/*
* Return the platform handle for the lgroup containing the given CPU
*/
/* ARGSUSED */
lgrp_handle_t
lgrp_plat_cpu_to_hand(processorid_t id)
{
if (lgrp_plat_node_cnt == 1)
return (LGRP_DEFAULT_HANDLE);
return ((lgrp_handle_t)LGRP_PLAT_CPU_TO_NODE(cpu[id]));
}
/*
* Return the platform handle of the lgroup that contains the physical memory
* corresponding to the given page frame number
*/
/* ARGSUSED */
lgrp_handle_t
lgrp_plat_pfn_to_hand(pfn_t pfn)
{
int mnode;
if (max_mem_nodes == 1)
return (LGRP_DEFAULT_HANDLE);
mnode = plat_pfn_to_mem_node(pfn);
return (MEM_NODE_2_LGRPHAND(mnode));
}
/*
* Return the maximum number of lgrps supported by the platform.
* Before lgrp topology is known it returns an estimate based on the number of
* nodes. Once topology is known it returns the actual maximim number of lgrps
* created. Since x86 doesn't support dynamic addition of new nodes, this number
* may not grow during system lifetime.
*/
int
lgrp_plat_max_lgrps()
{
return (lgrp_topo_initialized ?
lgrp_alloc_max + 1 :
lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
}
/*
* Return the number of free, allocatable, or installed
* pages in an lgroup
* This is a copy of the MAX_MEM_NODES == 1 version of the routine
* used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
*/
/* ARGSUSED */
static pgcnt_t
lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
{
struct memlist *mlist;
pgcnt_t npgs = 0;
extern struct memlist *phys_avail;
extern struct memlist *phys_install;
switch (query) {
case LGRP_MEM_SIZE_FREE:
return ((pgcnt_t)freemem);
case LGRP_MEM_SIZE_AVAIL:
memlist_read_lock();
for (mlist = phys_avail; mlist; mlist = mlist->next)
npgs += btop(mlist->size);
memlist_read_unlock();
return (npgs);
case LGRP_MEM_SIZE_INSTALL:
memlist_read_lock();
for (mlist = phys_install; mlist; mlist = mlist->next)
npgs += btop(mlist->size);
memlist_read_unlock();
return (npgs);
default:
return ((pgcnt_t)0);
}
}
/*
* Return the number of free pages in an lgroup.
*
* For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
* pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the
* number of allocatable base pagesize pages corresponding to the
* lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
* For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
* memory installed, regardless of whether or not it's usable.
*/
pgcnt_t
lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
{
int mnode;
pgcnt_t npgs = (pgcnt_t)0;
extern struct memlist *phys_avail;
extern struct memlist *phys_install;
if (plathand == LGRP_DEFAULT_HANDLE)
return (lgrp_plat_mem_size_default(plathand, query));
if (plathand != LGRP_NULL_HANDLE) {
mnode = plat_lgrphand_to_mem_node(plathand);
if (mnode >= 0 && mem_node_config[mnode].exists) {
switch (query) {
case LGRP_MEM_SIZE_FREE:
npgs = mem_node_config[mnode].cursize;
break;
case LGRP_MEM_SIZE_AVAIL:
npgs = mem_node_memlist_pages(mnode,
phys_avail);
break;
case LGRP_MEM_SIZE_INSTALL:
npgs = mem_node_memlist_pages(mnode,
phys_install);
break;
default:
break;
}
}
}
return (npgs);
}
/*
* Return latency between "from" and "to" lgroups
*
* This latency number can only be used for relative comparison
* between lgroups on the running system, cannot be used across platforms,
* and may not reflect the actual latency. It is platform and implementation
* specific, so platform gets to decide its value. It would be nice if the
* number was at least proportional to make comparisons more meaningful though.
*/
/* ARGSUSED */
int
lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
{
lgrp_handle_t src, dest;
if (max_mem_nodes == 1)
return (0);
/*
* Return max latency for root lgroup
*/
if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
return (lgrp_plat_probe_time_max);
src = from;
dest = to;
/*
* Return 0 for nodes (lgroup platform handles) out of range
*/
if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES)
return (0);
/*
* Probe from current CPU if its lgroup latencies haven't been set yet
* and we are trying to get latency from current CPU to some node
*/
if (lgrp_plat_probe_times[src][src] == 0 &&
LGRP_PLAT_CPU_TO_NODE(CPU) == src)
lgrp_plat_probe();
return (lgrp_plat_probe_times[src][dest]);
}
/*
* Return platform handle for root lgroup
*/
lgrp_handle_t
lgrp_plat_root_hand(void)
{
return (LGRP_DEFAULT_HANDLE);
}