Lines Matching defs:pg
89 cmt_lgrp_t *cmt_root = NULL; /* Reference to root cmt pg */
134 #define IS_CMT_PG(pg) (((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id)
228 pg_cmt_free(pg_t *pg)
230 ASSERT(pg != NULL);
231 ASSERT(IS_CMT_PG(pg));
233 kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t));
300 cmt_callback_init(pg_t *pg)
306 if (((pg_cmt_t *)pg)->cmt_policy == CMT_NO_POLICY)
309 switch (((pghw_t *)pg)->pghw_hw) {
311 pg->pg_cb.thread_swtch = cmt_ev_thread_swtch_pwr;
312 pg->pg_cb.thread_remain = cmt_ev_thread_remain_pwr;
315 pg->pg_cb.thread_swtch = cmt_ev_thread_swtch;
331 cmt_hier_promote(pg_cmt_t *pg, cpu_pg_t *pgdata)
344 parent = pg->cmt_parent;
352 ASSERT(PG_NUM_CPUS((pg_t *)pg) >= PG_NUM_CPUS((pg_t *)parent));
367 r = group_add(parent->cmt_siblings, pg, GRP_NORESIZE);
380 r = group_add(&cmt_root->cl_pgs, pg, GRP_NORESIZE);
393 if (group_remove(parent->cmt_children, pg, GRP_NORESIZE) != -1) {
398 children = pg->cmt_children;
399 pg->cmt_children = parent->cmt_children;
402 nchildren = pg->cmt_nchildren;
403 pg->cmt_nchildren = parent->cmt_nchildren;
409 pg->cmt_siblings = parent->cmt_siblings;
410 parent->cmt_siblings = pg->cmt_children;
413 * Update any cached lineages in the per CPU pg data.
415 PG_CPU_ITR_INIT(pg, cpu_iter);
437 if (cpu_pg->cmt_parent == pg) {
445 if ((idx = group_find(&pgd->cmt_pgs, (void *)pg)) == -1) {
466 err = group_add_at(&pgd->cmt_pgs, pg, idx - 1);
481 pg->cmt_parent = parent->cmt_parent;
482 parent->cmt_parent = pg;
502 pg_cmt_t *pg;
558 pg = (pg_cmt_t *)pghw_place_cpu(cp, hw);
559 if (pg == NULL) {
564 pg = (pg_cmt_t *)pg_create(pg_cmt_class_id);
567 pghw_init((pghw_t *)pg, cp, hw);
573 pg->cmt_policy = policy;
576 cmt_callback_init((pg_t *)pg);
578 bitset_init(&pg->cmt_cpus_actv_set);
579 group_create(&pg->cmt_cpus_actv);
581 ASSERT(IS_CMT_PG(pg));
584 ((pghw_t *)pg)->pghw_generation++;
587 pg_cpu_add((pg_t *)pg, cp, pgdata);
592 group_expand(&pg->cmt_cpus_actv,
593 GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
596 bitset_capacity(&pg->cmt_cpus_actv_set)) {
597 bitset_resize(&pg->cmt_cpus_actv_set,
605 cpu_cmt_hier[levels++] = pg;
610 pg_cache = (pg_t *)pg;
661 pg = cpu_cmt_hier[level];
666 while (pg->cmt_parent &&
667 pg->cmt_parent != cpu_cmt_hier[level + 1]) {
668 cmt_hier_promote(pg, pgdata);
688 pg = cpu_cmt_hier[level];
689 err = group_add_at(cmt_pgs, pg, levels - level - 1);
693 pgdata->cmt_lineage = (pg_t *)pg;
695 if (pg->cmt_siblings != NULL) {
697 ASSERT(pg->cmt_parent == NULL ||
698 pg->cmt_parent == cpu_cmt_hier[level + 1]);
699 ASSERT(pg->cmt_siblings == &lgrp->cl_pgs ||
700 ((pg->cmt_parent != NULL) &&
701 pg->cmt_siblings == pg->cmt_parent->cmt_children));
706 pg->cmt_parent = NULL;
708 pg->cmt_siblings = &lgrp->cl_pgs;
713 pg->cmt_parent = cpu_cmt_hier[level + 1];
720 if (pg->cmt_parent->cmt_children == NULL) {
721 pg->cmt_parent->cmt_children =
723 group_create(pg->cmt_parent->cmt_children);
725 pg->cmt_siblings = pg->cmt_parent->cmt_children;
726 children = ++pg->cmt_parent->cmt_nchildren;
729 group_expand(pg->cmt_siblings, children);
772 pg_cmt_t *pg;
818 pg = (pg_cmt_t *)pgdata->cmt_lineage;
819 while (pg != NULL) {
821 ((pghw_t *)pg)->pghw_generation++;
826 (void) group_remove(cmt_pgs, pg, GRP_RESIZE);
834 if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) {
835 if (pg->cmt_children != NULL)
836 group_destroy(pg->cmt_children);
837 if (pg->cmt_siblings != NULL) {
838 if (pg->cmt_siblings == &lgrp->cl_pgs)
841 pg->cmt_parent->cmt_nchildren--;
844 pg = pg->cmt_parent;
854 while ((pg = group_iterate(pgs, &i)) != NULL) {
855 if (IS_CMT_PG(pg) == 0)
858 pg_cpu_delete((pg_t *)pg, cp, pgdata);
866 if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) {
871 group_destroy(&pg->cmt_cpus_actv);
872 bitset_fini(&pg->cmt_cpus_actv_set);
873 pghw_fini((pghw_t *)pg);
875 pg_destroy((pg_t *)pg);
887 pg_t *pg;
903 while ((pg = group_iterate(pgs, &i)) != NULL) {
904 if (IS_CMT_PG(pg) == 0)
907 if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id)
908 bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1);
920 pg_t *pg;
936 while ((pg = group_iterate(pgs, &pg_iter)) != NULL) {
938 if (IS_CMT_PG(pg) == 0)
944 bitset_add(&newpp->cp_cmt_pgs, pg->pg_id);
951 PG_CPU_ITR_INIT(pg, cpu_iter);
962 bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id);
976 pg_cmt_t *pg;
990 while ((pg = group_iterate(pgs, &i)) != NULL) {
992 if (IS_CMT_PG(pg) == 0)
998 ((pghw_t *)pg)->pghw_generation++;
1000 err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
1009 if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 &&
1010 (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) {
1011 err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE);
1018 if (pg->cmt_parent == NULL &&
1019 pg->cmt_siblings != &cmt_root->cl_pgs) {
1020 err = group_add(&cmt_root->cl_pgs, pg,
1031 bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
1032 bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id);
1046 pg_cmt_t *pg;
1060 while ((pg = group_iterate(pgs, &i)) != NULL) {
1062 if (IS_CMT_PG(pg) == 0)
1068 ((pghw_t *)pg)->pghw_generation++;
1074 err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
1077 bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
1083 if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 &&
1084 (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) {
1085 err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
1088 if (pg->cmt_parent == NULL &&
1089 pg->cmt_siblings != &cmt_root->cl_pgs) {
1090 err = group_remove(&cmt_root->cl_pgs, pg,
1100 ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <=
1101 GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
1107 PG_CPU_ITR_INIT(pg, cpu_itr);
1119 ((pg_t *)pg)->pg_id);
1128 pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp)
1132 pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0);
1141 if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw))
1261 pg_cmt_t *pg;
1290 while ((pg = group_iterate(hwset, &iter)) != NULL) {
1296 pg->cmt_policy = pg_cmt_policy(((pghw_t *)pg)->pghw_hw);
1297 while ((pg->cmt_parent != NULL) &&
1298 (pg->cmt_parent->cmt_policy != pg->cmt_policy) &&
1299 (PG_NUM_CPUS((pg_t *)pg) ==
1300 PG_NUM_CPUS((pg_t *)pg->cmt_parent))) {
1301 cmt_hier_promote(pg, NULL);
1313 pg_cmt_t *pg;
1334 while ((pg = group_iterate(hwset, &iter)) != NULL) {
1341 if (pg->cmt_children != NULL &&
1342 GROUP_SIZE(pg->cmt_children) == 1) {
1343 child = GROUP_ACCESS(pg->cmt_children, 0);
1348 pg->cmt_policy = CMT_BALANCE;
1355 cmt_ev_thread_swtch(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old,
1358 pg_cmt_t *cmt_pg = (pg_cmt_t *)pg;
1370 #define THREAD_RUNNABLE_IN_PG(t, pg) \
1373 bitset_in_set(&(pg)->cmt_cpus_actv_set, \
1377 cmt_ev_thread_swtch_pwr(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old,
1380 pg_cmt_t *cmt = (pg_cmt_t *)pg;
1418 cmt_ev_thread_remain_pwr(pg_t *pg, cpu_t *cp, kthread_t *t)
1420 pg_cmt_t *cmt = (pg_cmt_t *)pg;
1432 pg_cmt_policy_name(pg_t *pg)
1436 policy = ((pg_cmt_t *)pg)->cmt_policy;
1471 pg_cmt_t *pg, *child;
1507 * represented by pg. But before we do that (and pause CPUs) we need
1528 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {
1534 if (pg->cmt_children &&
1535 GROUP_SIZE(pg->cmt_children) > 1) {
1536 cap_needed = GROUP_SIZE(pg->cmt_children) - 1;
1538 group_expand(pg->cmt_siblings,
1539 GROUP_SIZE(pg->cmt_siblings) + cap_needed);
1545 if (pg->cmt_parent == NULL &&
1546 pg->cmt_siblings != &cmt_root->cl_pgs) {
1562 * represented by pg.
1565 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {
1570 if (pg->cmt_siblings) {
1571 (void) group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
1573 if (pg->cmt_parent == NULL &&
1574 pg->cmt_siblings != &cmt_root->cl_pgs) {
1575 (void) group_remove(&cmt_root->cl_pgs, pg,
1583 pg->cmt_policy = CMT_NO_POLICY;
1595 if (pg->cmt_children != NULL) {
1596 children = pg->cmt_children;
1601 if (pg->cmt_siblings != NULL) {
1602 r = group_add(pg->cmt_siblings, child,
1606 if (pg->cmt_parent == NULL &&
1607 pg->cmt_siblings !=
1615 group_empty(pg->cmt_children);
1621 pg_callback_set_defaults((pg_t *)pg);
1626 PG_CPU_ITR_INIT(pg, cpu_iter);
1649 if (cpu_pg->cmt_parent == pg) {
1650 cpu_pg->cmt_parent = pg->cmt_parent;
1651 cpu_pg->cmt_siblings = pg->cmt_siblings;
1661 (void) group_remove(&cpd->cmt_pgs, pg, GRP_NORESIZE);
1744 pg_cmt_t *pg, *pg_next, *pg_bad, *pg_tmp, *parent;
1757 pg = lineage[i];
1768 (PG_NUM_CPUS((pg_t *)pg) <= PG_NUM_CPUS((pg_t *)pg_next)));
1792 parent = pg->cmt_parent;
1817 PG_NUM_CPUS((pg_t *)pg)) {
1832 PG_CPU_ITR_INIT((pg_t *)pg, cpu_iter);
1887 if (((pghw_t *)pg)->pghw_hw == PGHW_CHIP) {
1888 if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) {
1900 if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) {
1901 if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) {
1932 if ((PG_NUM_CPUS((pg_t *)pg) < PG_NUM_CPUS((pg_t *)pg_next)) &&
1933 (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw))) {
1934 pg_bad = pg;
1935 } else if (PG_NUM_CPUS((pg_t *)pg) ==
1939 } else if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) {
1940 pg_bad = pg;