common/cpr/cpr_dump.c

	cpr_dump.c revision 56f33205c9ed776c3c909e07d52e94610a675740
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Fill in and write out the cpr state file
 *  1. Allocate and write headers, ELF and cpr dump header
 *  2. Allocate bitmaps according to phys_install
 *  3. Tag kernel pages into corresponding bitmap
 *  4. Write bitmaps to state file
 *  5. Write actual physical page data to state file
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/vm.h>
#include <sys/memlist.h>
#include <sys/kmem.h>
#include <sys/vnode.h>
#include <sys/fs/ufs_inode.h>
#include <sys/errno.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <vm/page.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/hat.h>
#include <sys/cpr.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/panic.h>
#include <sys/thread.h>
#include <sys/note.h>

/* Local defines and variables */
#define BTOb(bytes) ((bytes) << 3)      /* Bytes to bits, log2(NBBY) */
#define bTOB(bits)  ((bits) >> 3)       /* bits to Bytes, log2(NBBY) */

#if defined(__sparc)
static uint_t cpr_pages_tobe_dumped;
static uint_t cpr_regular_pgs_dumped;
static int cpr_dump_regular_pages(vnode_t *);
static int cpr_count_upages(int, bitfunc_t);
static int cpr_compress_and_write(vnode_t *, uint_t, pfn_t, pgcnt_t);
#endif

int cpr_flush_write(vnode_t *);

int cpr_contig_pages(vnode_t *, int);

void cpr_clear_bitmaps();

extern size_t cpr_get_devsize(dev_t);
extern int i_cpr_dump_setup(vnode_t *);
extern int i_cpr_blockzero(char *, char **, int *, vnode_t *);
extern int cpr_test_mode;
int cpr_setbit(pfn_t, int);
int cpr_clrbit(pfn_t, int);

ctrm_t cpr_term;

char *cpr_buf, *cpr_buf_end;
int cpr_buf_blocks;     /* size of cpr_buf in blocks */
size_t cpr_buf_size;        /* size of cpr_buf in bytes */
size_t cpr_bitmap_size;
int cpr_nbitmaps;

char *cpr_pagedata;     /* page buffer for compression / tmp copy */
size_t cpr_pagedata_size;   /* page buffer size in bytes */

#if defined(__sparc)
static char *cpr_wptr;      /* keep track of where to write to next */
static int cpr_file_bn;     /* cpr state-file block offset */
static int cpr_disk_writes_ok;
static size_t cpr_dev_space = 0;
#endif

char cpr_pagecopy[CPR_MAXCONTIG * MMU_PAGESIZE];

#if defined(__sparc)
/*
 * On some platforms bcopy may modify the thread structure
 * during bcopy (eg, to prevent cpu migration).  If the
 * range we are currently writing out includes our own
 * thread structure then it will be snapshotted by bcopy
 * including those modified members - and the updates made
 * on exit from bcopy will no longer be seen when we later
 * restore the mid-bcopy kthread_t.  So if the range we
 * need to copy overlaps with our thread structure we will
 * use a simple byte copy.
 */
void
cprbcopy(void *from, void *to, size_t bytes)
{
    extern int curthreadremapped;
    caddr_t kthrend;

    kthrend = (caddr_t)curthread + sizeof (kthread_t) - 1;
    if (curthreadremapped || (kthrend >= (caddr_t)from &&
        kthrend < (caddr_t)from + bytes + sizeof (kthread_t) - 1)) {
        caddr_t src = from, dst = to;

        while (bytes-- > 0)
            *dst++ = *src++;
    } else {
        bcopy(from, to, bytes);
    }
}

/*
 * Allocate pages for buffers used in writing out the statefile
 */
static int
cpr_alloc_bufs(void)
{
    char *allocerr = "Unable to allocate memory for cpr buffer";
    size_t size;

    /*
     * set the cpr write buffer size to at least the historic
     * size (128k) or large enough to store the both the early
     * set of statefile structures (well under 0x800) plus the
     * bitmaps, and roundup to the next pagesize.
     */
    size = PAGE_ROUNDUP(dbtob(4) + cpr_bitmap_size);
    cpr_buf_size = MAX(size, CPRBUFSZ);
    cpr_buf_blocks = btodb(cpr_buf_size);
    cpr_buf = kmem_alloc(cpr_buf_size, KM_NOSLEEP);
    if (cpr_buf == NULL) {
        cpr_err(CE_WARN, allocerr);
        return (ENOMEM);
    }
    cpr_buf_end = cpr_buf + cpr_buf_size;

    cpr_pagedata_size = mmu_ptob(CPR_MAXCONTIG + 1);
    cpr_pagedata = kmem_alloc(cpr_pagedata_size, KM_NOSLEEP);
    if (cpr_pagedata == NULL) {
        kmem_free(cpr_buf, cpr_buf_size);
        cpr_buf = NULL;
        cpr_err(CE_WARN, allocerr);
        return (ENOMEM);
    }

    return (0);
}


/*
 * Set bitmap size in bytes based on phys_install.
 */
void
cpr_set_bitmap_size(void)
{
    struct memlist *pmem;
    size_t size = 0;

    memlist_read_lock();
    for (pmem = phys_install; pmem; pmem = pmem->ml_next)
        size += pmem->ml_size;
    memlist_read_unlock();
    cpr_bitmap_size = BITMAP_BYTES(size);
}


/*
 * CPR dump header contains the following information:
 *  1. header magic -- unique to cpr state file
 *  2. kernel return pc & ppn for resume
 *  3. current thread info
 *  4. debug level and test mode
 *  5. number of bitmaps allocated
 *  6. number of page records
 */
static int
cpr_write_header(vnode_t *vp)
{
    extern ushort_t cpr_mach_type;
    struct cpr_dump_desc cdump;
    pgcnt_t bitmap_pages;
    pgcnt_t kpages, vpages, upages;
    pgcnt_t cpr_count_kpages(int mapflag, bitfunc_t bitfunc);

    cdump.cdd_magic = (uint_t)CPR_DUMP_MAGIC;
    cdump.cdd_version = CPR_VERSION;
    cdump.cdd_machine = cpr_mach_type;
    cdump.cdd_debug = cpr_debug;
    cdump.cdd_test_mode = cpr_test_mode;
    cdump.cdd_bitmaprec = cpr_nbitmaps;

    cpr_clear_bitmaps();

    /*
     * Remember how many pages we plan to save to statefile.
     * This information will be used for sanity checks.
     * Untag those pages that will not be saved to statefile.
     */
    kpages = cpr_count_kpages(REGULAR_BITMAP, cpr_setbit);
    vpages = cpr_count_volatile_pages(REGULAR_BITMAP, cpr_clrbit);
    upages = cpr_count_upages(REGULAR_BITMAP, cpr_setbit);
    cdump.cdd_dumppgsize = kpages - vpages + upages;
    cpr_pages_tobe_dumped = cdump.cdd_dumppgsize;
    CPR_DEBUG(CPR_DEBUG7,
        "\ncpr_write_header: kpages %ld - vpages %ld + upages %ld = %d\n",
        kpages, vpages, upages, cdump.cdd_dumppgsize);

    /*
     * Some pages contain volatile data (cpr_buf and storage area for
     * sensitive kpages), which are no longer needed after the statefile
     * is dumped to disk.  We have already untagged them from regular
     * bitmaps.  Now tag them into the volatile bitmaps.  The pages in
     * volatile bitmaps will be claimed during resume, and the resumed
     * kernel will free them.
     */
    (void) cpr_count_volatile_pages(VOLATILE_BITMAP, cpr_setbit);

    bitmap_pages = mmu_btopr(cpr_bitmap_size);

    /*
     * Export accurate statefile size for statefile allocation retry.
     * statefile_size = all the headers + total pages +
     * number of pages used by the bitmaps.
     * Roundup will be done in the file allocation code.
     */
    STAT->cs_nocomp_statefsz = sizeof (cdd_t) + sizeof (cmd_t) +
        (sizeof (cbd_t) * cdump.cdd_bitmaprec) +
        (sizeof (cpd_t) * cdump.cdd_dumppgsize) +
        mmu_ptob(cdump.cdd_dumppgsize + bitmap_pages);

    /*
     * If the estimated statefile is not big enough,
     * go retry now to save un-necessary operations.
     */
    if (!(CPR->c_flags & C_COMPRESSING) &&
        (STAT->cs_nocomp_statefsz > STAT->cs_est_statefsz)) {
        if (cpr_debug & (CPR_DEBUG1 | CPR_DEBUG7))
            prom_printf("cpr_write_header: "
                "STAT->cs_nocomp_statefsz > "
                "STAT->cs_est_statefsz\n");
        return (ENOSPC);
    }

    /* now write cpr dump descriptor */
    return (cpr_write(vp, (caddr_t)&cdump, sizeof (cdd_t)));
}


/*
 * CPR dump tail record contains the following information:
 *  1. header magic -- unique to cpr state file
 *  2. all misc info that needs to be passed to cprboot or resumed kernel
 */
static int
cpr_write_terminator(vnode_t *vp)
{
    cpr_term.magic = (uint_t)CPR_TERM_MAGIC;
    cpr_term.va = (cpr_ptr)&cpr_term;
    cpr_term.pfn = (cpr_ext)va_to_pfn(&cpr_term);

    /* count the last one (flush) */
    cpr_term.real_statef_size = STAT->cs_real_statefsz +
        btod(cpr_wptr - cpr_buf) * DEV_BSIZE;

    CPR_DEBUG(CPR_DEBUG9, "cpr_dump: Real Statefile Size: %ld\n",
        STAT->cs_real_statefsz);

    cpr_tod_get(&cpr_term.tm_shutdown);

    return (cpr_write(vp, (caddr_t)&cpr_term, sizeof (cpr_term)));
}

/*
 * Write bitmap descriptor array, followed by merged bitmaps.
 */
static int
cpr_write_bitmap(vnode_t *vp)
{
    char *rmap, *vmap, *dst, *tail;
    size_t size, bytes;
    cbd_t *dp;
    int err;

    dp = CPR->c_bmda;
    if (err = cpr_write(vp, (caddr_t)dp, cpr_nbitmaps * sizeof (*dp)))
        return (err);

    /*
     * merge regular and volatile bitmaps into tmp space
     * and write to disk
     */
    for (; dp->cbd_size; dp++) {
        rmap = (char *)dp->cbd_reg_bitmap;
        vmap = (char *)dp->cbd_vlt_bitmap;
        for (size = dp->cbd_size; size; size -= bytes) {
            bytes = min(size, sizeof (cpr_pagecopy));
            tail = &cpr_pagecopy[bytes];
            for (dst = cpr_pagecopy; dst < tail; dst++)
                *dst = *rmap++ | *vmap++;
            if (err = cpr_write(vp, cpr_pagecopy, bytes))
                break;
        }
    }

    return (err);
}


static int
cpr_write_statefile(vnode_t *vp)
{
    uint_t error = 0;
    extern  int i_cpr_check_pgs_dumped();
    void flush_windows(void);
    pgcnt_t spages;
    char *str;

    flush_windows();

    /*
     * to get an accurate view of kas, we need to untag sensitive
     * pages *before* dumping them because the disk driver makes
     * allocations and changes kas along the way.  The remaining
     * pages referenced in the bitmaps are dumped out later as
     * regular kpages.
     */
    str = "cpr_write_statefile:";
    spages = i_cpr_count_sensitive_kpages(REGULAR_BITMAP, cpr_clrbit);
    CPR_DEBUG(CPR_DEBUG7, "%s untag %ld sens pages\n", str, spages);

    /*
     * now it's OK to call a driver that makes allocations
     */
    cpr_disk_writes_ok = 1;

    /*
     * now write out the clean sensitive kpages
     * according to the sensitive descriptors
     */
    error = i_cpr_dump_sensitive_kpages(vp);
    if (error) {
        CPR_DEBUG(CPR_DEBUG7,
            "%s cpr_dump_sensitive_kpages() failed!\n", str);
        return (error);
    }

    /*
     * cpr_dump_regular_pages() counts cpr_regular_pgs_dumped
     */
    error = cpr_dump_regular_pages(vp);
    if (error) {
        CPR_DEBUG(CPR_DEBUG7,
            "%s cpr_dump_regular_pages() failed!\n", str);
        return (error);
    }

    /*
     * sanity check to verify the right number of pages were dumped
     */
    error = i_cpr_check_pgs_dumped(cpr_pages_tobe_dumped,
        cpr_regular_pgs_dumped);

    if (error) {
        prom_printf("\n%s page count mismatch!\n", str);
#ifdef DEBUG
        if (cpr_test_mode)
            debug_enter(NULL);
#endif
    }

    return (error);
}
#endif


/*
 * creates the CPR state file, the following sections are
 * written out in sequence:
 *    - writes the cpr dump header
 *    - writes the memory usage bitmaps
 *    - writes the platform dependent info
 *    - writes the remaining user pages
 *    - writes the kernel pages
 */
#if defined(__x86)
    _NOTE(ARGSUSED(0))
#endif
int
cpr_dump(vnode_t *vp)
{
#if defined(__sparc)
    int error;

    if (cpr_buf == NULL) {
        ASSERT(cpr_pagedata == NULL);
        if (error = cpr_alloc_bufs())
            return (error);
    }
    /* point to top of internal buffer */
    cpr_wptr = cpr_buf;

    /* initialize global variables used by the write operation */
    cpr_file_bn = cpr_statefile_offset();
    cpr_dev_space = 0;

    /* allocate bitmaps */
    if (CPR->c_bmda == NULL) {
        if (error = i_cpr_alloc_bitmaps()) {
            cpr_err(CE_WARN, "cannot allocate bitmaps");
            return (error);
        }
    }

    if (error = i_cpr_prom_pages(CPR_PROM_SAVE))
        return (error);

    if (error = i_cpr_dump_setup(vp))
        return (error);

    /*
     * set internal cross checking; we dont want to call
     * a disk driver that makes allocations until after
     * sensitive pages are saved
     */
    cpr_disk_writes_ok = 0;

    /*
     * 1253112: heap corruption due to memory allocation when dumpping
     *      statefile.
     * Theoretically on Sun4u only the kernel data nucleus, kvalloc and
     * kvseg segments can be contaminated should memory allocations happen
     * during sddump, which is not supposed to happen after the system
     * is quiesced. Let's call the kernel pages that tend to be affected
     * 'sensitive kpages' here. To avoid saving inconsistent pages, we
     * will allocate some storage space to save the clean sensitive pages
     * aside before statefile dumping takes place. Since there may not be
     * much memory left at this stage, the sensitive pages will be
     * compressed before they are saved into the storage area.
     */
    if (error = i_cpr_save_sensitive_kpages()) {
        CPR_DEBUG(CPR_DEBUG7,
            "cpr_dump: save_sensitive_kpages failed!\n");
        return (error);
    }

    /*
     * since all cpr allocations are done (space for sensitive kpages,
     * bitmaps, cpr_buf), kas is stable, and now we can accurately
     * count regular and sensitive kpages.
     */
    if (error = cpr_write_header(vp)) {
        CPR_DEBUG(CPR_DEBUG7,
            "cpr_dump: cpr_write_header() failed!\n");
        return (error);
    }

    if (error = i_cpr_write_machdep(vp))
        return (error);

    if (error = i_cpr_blockzero(cpr_buf, &cpr_wptr, NULL, NULL))
        return (error);

    if (error = cpr_write_bitmap(vp))
        return (error);

    if (error = cpr_write_statefile(vp)) {
        CPR_DEBUG(CPR_DEBUG7,
            "cpr_dump: cpr_write_statefile() failed!\n");
        return (error);
    }

    if (error = cpr_write_terminator(vp))
        return (error);

    if (error = cpr_flush_write(vp))
        return (error);

    if (error = i_cpr_blockzero(cpr_buf, &cpr_wptr, &cpr_file_bn, vp))
        return (error);
#endif

    return (0);
}


#if defined(__sparc)
/*
 * cpr_xwalk() is called many 100x with a range within kvseg or kvseg_reloc;
 * a page-count from each range is accumulated at arg->pages.
 */
static void
cpr_xwalk(void *arg, void *base, size_t size)
{
    struct cpr_walkinfo *cwip = arg;

    cwip->pages += cpr_count_pages(base, size,
        cwip->mapflag, cwip->bitfunc, DBG_DONTSHOWRANGE);
    cwip->size += size;
    cwip->ranges++;
}

/*
 * cpr_walk() is called many 100x with a range within kvseg or kvseg_reloc;
 * a page-count from each range is accumulated at arg->pages.
 */
static void
cpr_walk(void *arg, void *base, size_t size)
{
    caddr_t addr = base;
    caddr_t addr_end = addr + size;

    /*
     * If we are about to start walking the range of addresses we
     * carved out of the kernel heap for the large page heap walk
     * heap_lp_arena to find what segments are actually populated
     */
    if (SEGKMEM_USE_LARGEPAGES &&
        addr == heap_lp_base && addr_end == heap_lp_end &&
        vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
        vmem_walk(heap_lp_arena, VMEM_ALLOC, cpr_xwalk, arg);
    } else {
        cpr_xwalk(arg, base, size);
    }
}


/*
 * faster scan of kvseg using vmem_walk() to visit
 * allocated ranges.
 */
pgcnt_t
cpr_scan_kvseg(int mapflag, bitfunc_t bitfunc, struct seg *seg)
{
    struct cpr_walkinfo cwinfo;

    bzero(&cwinfo, sizeof (cwinfo));
    cwinfo.mapflag = mapflag;
    cwinfo.bitfunc = bitfunc;

    vmem_walk(heap_arena, VMEM_ALLOC, cpr_walk, &cwinfo);

    if (cpr_debug & CPR_DEBUG7) {
        prom_printf("walked %d sub-ranges, total pages %ld\n",
            cwinfo.ranges, mmu_btop(cwinfo.size));
        cpr_show_range(seg->s_base, seg->s_size,
            mapflag, bitfunc, cwinfo.pages);
    }

    return (cwinfo.pages);
}


/*
 * cpr_walk_kpm() is called for every used area within the large
 * segkpm virtual address window. A page-count is accumulated at
 * arg->pages.
 */
static void
cpr_walk_kpm(void *arg, void *base, size_t size)
{
    struct cpr_walkinfo *cwip = arg;

    cwip->pages += cpr_count_pages(base, size,
        cwip->mapflag, cwip->bitfunc, DBG_DONTSHOWRANGE);
    cwip->size += size;
    cwip->ranges++;
}


/*
 * faster scan of segkpm using hat_kpm_walk() to visit only used ranges.
 */
/*ARGSUSED*/
static pgcnt_t
cpr_scan_segkpm(int mapflag, bitfunc_t bitfunc, struct seg *seg)
{
    struct cpr_walkinfo cwinfo;

    if (kpm_enable == 0)
        return (0);

    bzero(&cwinfo, sizeof (cwinfo));
    cwinfo.mapflag = mapflag;
    cwinfo.bitfunc = bitfunc;
    hat_kpm_walk(cpr_walk_kpm, &cwinfo);

    if (cpr_debug & CPR_DEBUG7) {
        prom_printf("walked %d sub-ranges, total pages %ld\n",
            cwinfo.ranges, mmu_btop(cwinfo.size));
        cpr_show_range(segkpm->s_base, segkpm->s_size,
            mapflag, bitfunc, cwinfo.pages);
    }

    return (cwinfo.pages);
}


/*
 * Sparsely filled kernel segments are registered in kseg_table for
 * easier lookup. See also block comment for cpr_count_seg_pages.
 */

#define KSEG_SEG_ADDR   0   /* address of struct seg */
#define KSEG_PTR_ADDR   1   /* address of pointer to struct seg */

typedef struct {
    struct seg **st_seg;        /* segment pointer or segment address */
    pgcnt_t (*st_fcn)(int, bitfunc_t, struct seg *); /* function to call */
    int st_addrtype;        /* address type in st_seg */
} ksegtbl_entry_t;

ksegtbl_entry_t kseg_table[] = {
    {(struct seg **)&kvseg,     cpr_scan_kvseg,     KSEG_SEG_ADDR},
    {&segkpm,           cpr_scan_segkpm,    KSEG_PTR_ADDR},
    {NULL,              0,          0}
};


/*
 * Compare seg with each entry in kseg_table; when there is a match
 * return the entry pointer, otherwise return NULL.
 */
static ksegtbl_entry_t *
cpr_sparse_seg_check(struct seg *seg)
{
    ksegtbl_entry_t *ste = &kseg_table[0];
    struct seg *tseg;

    for (; ste->st_seg; ste++) {
        tseg = (ste->st_addrtype == KSEG_PTR_ADDR) ?
            *ste->st_seg : (struct seg *)ste->st_seg;

        if (seg == tseg)
            return (ste);
    }

    return ((ksegtbl_entry_t *)NULL);
}


/*
 * Count pages within each kernel segment; call cpr_sparse_seg_check()
 * to find out whether a sparsely filled segment needs special
 * treatment (e.g. kvseg).
 * Todo: A "SEGOP_CPR" like SEGOP_DUMP should be introduced, the cpr
 *       module shouldn't need to know segment details like if it is
 *       sparsely filled or not (makes kseg_table obsolete).
 */
pgcnt_t
cpr_count_seg_pages(int mapflag, bitfunc_t bitfunc)
{
    struct seg *segp;
    pgcnt_t pages;
    ksegtbl_entry_t *ste;

    pages = 0;
    for (segp = AS_SEGFIRST(&kas); segp; segp = AS_SEGNEXT(&kas, segp)) {
        if (ste = cpr_sparse_seg_check(segp)) {
            pages += (ste->st_fcn)(mapflag, bitfunc, segp);
        } else {
            pages += cpr_count_pages(segp->s_base,
                segp->s_size, mapflag, bitfunc, DBG_SHOWRANGE);
        }
    }

    return (pages);
}


/*
 * count kernel pages within kas and any special ranges
 */
pgcnt_t
cpr_count_kpages(int mapflag, bitfunc_t bitfunc)
{
    pgcnt_t kas_cnt;

    /*
     * Some pages need to be taken care of differently.
     * eg: panicbuf pages of sun4m are not in kas but they need
     * to be saved.  On sun4u, the physical pages of panicbuf are
     * allocated via prom_retain().
     */
    kas_cnt = i_cpr_count_special_kpages(mapflag, bitfunc);
    kas_cnt += cpr_count_seg_pages(mapflag, bitfunc);

    CPR_DEBUG(CPR_DEBUG9, "cpr_count_kpages: kas_cnt=%ld\n", kas_cnt);
    CPR_DEBUG(CPR_DEBUG7, "\ncpr_count_kpages: %ld pages, 0x%lx bytes\n",
        kas_cnt, mmu_ptob(kas_cnt));

    return (kas_cnt);
}


/*
 * Set a bit corresponding to the arg phys page number;
 * returns 0 when the ppn is valid and the corresponding
 * map bit was clear, otherwise returns 1.
 */
int
cpr_setbit(pfn_t ppn, int mapflag)
{
    char *bitmap;
    cbd_t *dp;
    pfn_t rel;
    int clr;

    for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
        if (PPN_IN_RANGE(ppn, dp)) {
            bitmap = DESC_TO_MAP(dp, mapflag);
            rel = ppn - dp->cbd_spfn;
            if ((clr = isclr(bitmap, rel)) != 0)
                setbit(bitmap, rel);
            return (clr == 0);
        }
    }

    return (1);
}


/*
 * Clear a bit corresponding to the arg phys page number.
 */
int
cpr_clrbit(pfn_t ppn, int mapflag)
{
    char *bitmap;
    cbd_t *dp;
    pfn_t rel;
    int set;

    for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
        if (PPN_IN_RANGE(ppn, dp)) {
            bitmap = DESC_TO_MAP(dp, mapflag);
            rel = ppn - dp->cbd_spfn;
            if ((set = isset(bitmap, rel)) != 0)
                clrbit(bitmap, rel);
            return (set == 0);
        }
    }

    return (1);
}


/* ARGSUSED */
int
cpr_nobit(pfn_t ppn, int mapflag)
{
    return (0);
}


/*
 * Lookup a bit corresponding to the arg phys page number.
 */
int
cpr_isset(pfn_t ppn, int mapflag)
{
    char *bitmap;
    cbd_t *dp;
    pfn_t rel;

    for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
        if (PPN_IN_RANGE(ppn, dp)) {
            bitmap = DESC_TO_MAP(dp, mapflag);
            rel = ppn - dp->cbd_spfn;
            return (isset(bitmap, rel));
        }
    }

    return (0);
}


/*
 * Go thru all pages and pick up any page not caught during the invalidation
 * stage. This is also used to save pages with cow lock or phys page lock held
 * (none zero p_lckcnt or p_cowcnt)
 */
static  int
cpr_count_upages(int mapflag, bitfunc_t bitfunc)
{
    page_t *pp, *page0;
    pgcnt_t dcnt = 0, tcnt = 0;
    pfn_t pfn;

    page0 = pp = page_first();

    do {
        if (pp->p_vnode == NULL || PP_ISKAS(pp) ||
            PP_ISFREE(pp) && PP_ISAGED(pp))
            continue;

        pfn = page_pptonum(pp);
        if (pf_is_memory(pfn)) {
            tcnt++;
            if ((*bitfunc)(pfn, mapflag) == 0)
                dcnt++; /* dirty count */
        }
    } while ((pp = page_next(pp)) != page0);

    STAT->cs_upage2statef = dcnt;
    CPR_DEBUG(CPR_DEBUG9, "cpr_count_upages: dirty=%ld total=%ld\n",
        dcnt, tcnt);
    CPR_DEBUG(CPR_DEBUG7, "cpr_count_upages: %ld pages, 0x%lx bytes\n",
        dcnt, mmu_ptob(dcnt));
    page0 = NULL; /* for Lint */
    return (dcnt);
}


/*
 * try compressing pages based on cflag,
 * and for DEBUG kernels, verify uncompressed data checksum;
 *
 * this routine replaces common code from
 * i_cpr_compress_and_save() and cpr_compress_and_write()
 */
char *
cpr_compress_pages(cpd_t *dp, pgcnt_t pages, int cflag)
{
    size_t nbytes, clen, len;
    uint32_t test_sum;
    char *datap;

    nbytes = mmu_ptob(pages);

    /*
     * set length to the original uncompressed data size;
     * always init cpd_flag to zero
     */
    dp->cpd_length = nbytes;
    dp->cpd_flag = 0;

#ifdef  DEBUG
    /*
     * Make a copy of the uncompressed data so we can checksum it.
     * Compress that copy so the checksum works at the other end
     */
    cprbcopy(CPR->c_mapping_area, cpr_pagecopy, nbytes);
    dp->cpd_usum = checksum32(cpr_pagecopy, nbytes);
    dp->cpd_flag |= CPD_USUM;
    datap = cpr_pagecopy;
#else
    datap = CPR->c_mapping_area;
    dp->cpd_usum = 0;
#endif

    /*
     * try compressing the raw data to cpr_pagedata;
     * if there was a size reduction: record the new length,
     * flag the compression, and point to the compressed data.
     */
    dp->cpd_csum = 0;
    if (cflag) {
        clen = compress(datap, cpr_pagedata, nbytes);
        if (clen < nbytes) {
            dp->cpd_flag |= CPD_COMPRESS;
            dp->cpd_length = clen;
            datap = cpr_pagedata;
#ifdef  DEBUG
            dp->cpd_csum = checksum32(datap, clen);
            dp->cpd_flag |= CPD_CSUM;

            /*
             * decompress the data back to a scratch area
             * and compare the new checksum with the original
             * checksum to verify the compression.
             */
            bzero(cpr_pagecopy, sizeof (cpr_pagecopy));
            len = decompress(datap, cpr_pagecopy,
                clen, sizeof (cpr_pagecopy));
            test_sum = checksum32(cpr_pagecopy, len);
            ASSERT(test_sum == dp->cpd_usum);
#endif
        }
    }

    return (datap);
}


/*
 * 1. Prepare cpr page descriptor and write it to file
 * 2. Compress page data and write it out
 */
static int
cpr_compress_and_write(vnode_t *vp, uint_t va, pfn_t pfn, pgcnt_t npg)
{
    int error = 0;
    char *datap;
    cpd_t cpd;  /* cpr page descriptor */
    extern void i_cpr_mapin(caddr_t, uint_t, pfn_t);
    extern void i_cpr_mapout(caddr_t, uint_t);

    i_cpr_mapin(CPR->c_mapping_area, npg, pfn);

    CPR_DEBUG(CPR_DEBUG3, "mapped-in %ld pages, vaddr 0x%p, pfn 0x%lx\n",
        npg, (void *)CPR->c_mapping_area, pfn);

    /*
     * Fill cpr page descriptor.
     */
    cpd.cpd_magic = (uint_t)CPR_PAGE_MAGIC;
    cpd.cpd_pfn = pfn;
    cpd.cpd_pages = npg;

    STAT->cs_dumped_statefsz += mmu_ptob(npg);

    datap = cpr_compress_pages(&cpd, npg, CPR->c_flags & C_COMPRESSING);

    /* Write cpr page descriptor */
    error = cpr_write(vp, (caddr_t)&cpd, sizeof (cpd_t));

    /* Write compressed page data */
    error = cpr_write(vp, (caddr_t)datap, cpd.cpd_length);

    /*
     * Unmap the pages for tlb and vac flushing
     */
    i_cpr_mapout(CPR->c_mapping_area, npg);

    if (error) {
        CPR_DEBUG(CPR_DEBUG1,
            "cpr_compress_and_write: vp 0x%p va 0x%x ", (void *)vp, va);
        CPR_DEBUG(CPR_DEBUG1, "pfn 0x%lx blk %d err %d\n",
            pfn, cpr_file_bn, error);
    } else {
        cpr_regular_pgs_dumped += npg;
    }

    return (error);
}


int
cpr_write(vnode_t *vp, caddr_t buffer, size_t size)
{
    caddr_t fromp = buffer;
    size_t bytes, wbytes;
    int error;

    if (cpr_dev_space == 0) {
        if (vp->v_type == VBLK) {
            cpr_dev_space = cpr_get_devsize(vp->v_rdev);
            ASSERT(cpr_dev_space);
        } else
            cpr_dev_space = 1;  /* not used in this case */
    }

    /*
     * break the write into multiple part if request is large,
     * calculate count up to buf page boundary, then write it out.
     * repeat until done.
     */
    while (size) {
        bytes = MIN(size, cpr_buf_end - cpr_wptr);
        cprbcopy(fromp, cpr_wptr, bytes);
        cpr_wptr += bytes;
        fromp += bytes;
        size -= bytes;
        if (cpr_wptr < cpr_buf_end)
            return (0); /* buffer not full yet */
        ASSERT(cpr_wptr == cpr_buf_end);

        wbytes = dbtob(cpr_file_bn + cpr_buf_blocks);
        if (vp->v_type == VBLK) {
            if (wbytes > cpr_dev_space)
                return (ENOSPC);
        } else {
            if (wbytes > VTOI(vp)->i_size)
                return (ENOSPC);
        }

        CPR_DEBUG(CPR_DEBUG3,
            "cpr_write: frmp=%p wptr=%p cnt=%lx...",
            (void *)fromp, (void *)cpr_wptr, bytes);
        /*
         * cross check, this should not happen!
         */
        if (cpr_disk_writes_ok == 0) {
            prom_printf("cpr_write: disk write too early!\n");
            return (EINVAL);
        }

        do_polled_io = 1;
        error = VOP_DUMP(vp, cpr_buf, cpr_file_bn, cpr_buf_blocks,
            NULL);
        do_polled_io = 0;
        CPR_DEBUG(CPR_DEBUG3, "done\n");

        STAT->cs_real_statefsz += cpr_buf_size;

        if (error) {
            cpr_err(CE_WARN, "cpr_write error %d", error);
            return (error);
        }
        cpr_file_bn += cpr_buf_blocks;  /* Increment block count */
        cpr_wptr = cpr_buf;     /* back to top of buffer */
    }
    return (0);
}


int
cpr_flush_write(vnode_t *vp)
{
    int nblk;
    int error;

    /*
     * Calculate remaining blocks in buffer, rounded up to nearest
     * disk block
     */
    nblk = btod(cpr_wptr - cpr_buf);

    do_polled_io = 1;
    error = VOP_DUMP(vp, (caddr_t)cpr_buf, cpr_file_bn, nblk, NULL);
    do_polled_io = 0;

    cpr_file_bn += nblk;
    if (error)
        CPR_DEBUG(CPR_DEBUG2, "cpr_flush_write: error (%d)\n",
            error);
    return (error);
}

void
cpr_clear_bitmaps(void)
{
    cbd_t *dp;

    for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
        bzero((void *)dp->cbd_reg_bitmap,
            (size_t)dp->cbd_size * 2);
    }
    CPR_DEBUG(CPR_DEBUG7, "\ncleared reg and vlt bitmaps\n");
}

int
cpr_contig_pages(vnode_t *vp, int flag)
{
    int chunks = 0, error = 0;
    pgcnt_t i, j, totbit;
    pfn_t spfn;
    cbd_t *dp;
    uint_t  spin_cnt = 0;
    extern  int i_cpr_compress_and_save();

    for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
        spfn = dp->cbd_spfn;
        totbit = BTOb(dp->cbd_size);
        i = 0; /* Beginning of bitmap */
        j = 0;
        while (i < totbit) {
            while ((j < CPR_MAXCONTIG) && ((j + i) < totbit)) {
                if (isset((char *)dp->cbd_reg_bitmap, j+i))
                    j++;
                else /* not contiguous anymore */
                    break;
            }

            if (j) {
                chunks++;
                if (flag == SAVE_TO_STORAGE) {
                    error = i_cpr_compress_and_save(
                        chunks, spfn + i, j);
                    if (error)
                        return (error);
                } else if (flag == WRITE_TO_STATEFILE) {
                    error = cpr_compress_and_write(vp, 0,
                        spfn + i, j);
                    if (error)
                        return (error);
                    else {
                        spin_cnt++;
                        if ((spin_cnt & 0x5F) == 1)
                            cpr_spinning_bar();
                    }
                }
            }

            i += j;
            if (j != CPR_MAXCONTIG) {
                /* Stopped on a non-tagged page */
                i++;
            }

            j = 0;
        }
    }

    if (flag == STORAGE_DESC_ALLOC)
        return (chunks);
    else
        return (0);
}


void
cpr_show_range(caddr_t vaddr, size_t size,
    int mapflag, bitfunc_t bitfunc, pgcnt_t count)
{
    char *action, *bname;

    bname = (mapflag == REGULAR_BITMAP) ? "regular" : "volatile";
    if (bitfunc == cpr_setbit)
        action = "tag";
    else if (bitfunc == cpr_clrbit)
        action = "untag";
    else
        action = "none";
    prom_printf("range (0x%p, 0x%p), %s bitmap, %s %ld\n",
        (void *)vaddr, (void *)(vaddr + size), bname, action, count);
}


pgcnt_t
cpr_count_pages(caddr_t sva, size_t size,
    int mapflag, bitfunc_t bitfunc, int showrange)
{
    caddr_t va, eva;
    pfn_t pfn;
    pgcnt_t count = 0;

    eva = sva + PAGE_ROUNDUP(size);
    for (va = sva; va < eva; va += MMU_PAGESIZE) {
        pfn = va_to_pfn(va);
        if (pfn != PFN_INVALID && pf_is_memory(pfn)) {
            if ((*bitfunc)(pfn, mapflag) == 0)
                count++;
        }
    }

    if ((cpr_debug & CPR_DEBUG7) && showrange == DBG_SHOWRANGE)
        cpr_show_range(sva, size, mapflag, bitfunc, count);

    return (count);
}


pgcnt_t
cpr_count_volatile_pages(int mapflag, bitfunc_t bitfunc)
{
    pgcnt_t count = 0;

    if (cpr_buf) {
        count += cpr_count_pages(cpr_buf, cpr_buf_size,
            mapflag, bitfunc, DBG_SHOWRANGE);
    }
    if (cpr_pagedata) {
        count += cpr_count_pages(cpr_pagedata, cpr_pagedata_size,
            mapflag, bitfunc, DBG_SHOWRANGE);
    }
    count += i_cpr_count_storage_pages(mapflag, bitfunc);

    CPR_DEBUG(CPR_DEBUG7, "cpr_count_vpages: %ld pages, 0x%lx bytes\n",
        count, mmu_ptob(count));
    return (count);
}


static int
cpr_dump_regular_pages(vnode_t *vp)
{
    int error;

    cpr_regular_pgs_dumped = 0;
    error = cpr_contig_pages(vp, WRITE_TO_STATEFILE);
    if (!error)
        CPR_DEBUG(CPR_DEBUG7, "cpr_dump_regular_pages() done.\n");
    return (error);
}
#endif