ndmpd/tlm/tlm_bitmap.c

	tlm_bitmap.c revision 47842382d52f28aa3173aa6b511781c322ccb6a2
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * BSD 3 Clause License
 *
 * Copyright (c) 2007, The Storage Networking Industry Association.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *  - Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 *  - Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 *  - Neither the name of The Storage Networking Industry Association (SNIA)
 *    nor the names of its contributors may be used to endorse or promote
 *    products derived from this software without specific prior written
 *    permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#include <sys/types.h>
#include <sys/queue.h>
#include <bitmap.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <tlm.h>


/*
 * Hash table size.
 */
#define BMAP_HASH_SIZE      64


/*
 * Maximum number of chunk that can be cached.
 */
#define BMAP_CHUNK_MAX      128


/*
 * Size of bitmap table.
 */
#define BMAP_MAX    256


/*
 * Bit_MAP Word SIZE.  This should be equal to 'sizeof (int)'.
 */
#define BMAP_WSIZE  (sizeof (int))


/*
 * Bit_MAP Bit Per Word.
 */
#define BMAP_BPW    (BMAP_WSIZE * 8)
#define BMAP_BPW_SHIFT  5
#define BMAP_BPW_MASK   (~(~0 << BMAP_BPW_SHIFT))


/*
 * Chunk of bit map in each node.
 */
#define BMAP_CHUNK_WORDS    1024
#define BMAP_CHUNK_BYTES    (BMAP_CHUNK_WORDS * BMAP_WSIZE)
#define BMAP_CHUNK_BITS     (BMAP_CHUNK_WORDS * BMAP_BPW)
#define BMAP_CHUNK_NO(p)    ((p) / BMAP_CHUNK_BITS)
#define BMAP_CHUNK_OFF(p)   (BMAP_CHUNK_NO(p) * BMAP_CHUNK_BITS)


/*
 * Bitmap flags.
 */
#define BMAP_BINIT_ONES 0x00000001 /* initial value of bits is 1 */
#define BMAP_INUSE  0x00000002 /* slot is in use */


/*
 * Macros of bitmap flags.
 */
#define BMAP_SET_FLAGS(b, f)    ((b)->bm_flags |= (f))
#define BMAP_UNSET_FLAGS(b, f)  ((b)->bm_flags &= ~(f))

#define BMAP_IS_INIT_ONES(b)    ((b)->bm_flags & BMAP_BINIT_ONES)
#define BMAP_IS_INUSE(b)    ((b)->bm_flags & BMAP_INUSE)


#define HASH(p)     (((p) / BMAP_CHUNK_BITS) % BMAP_HASH_SIZE)

/*
 * Calculate the memory size in bytes needed for the specified length
 * of bitmap.
 */
#define ROUNDUP(n, d)   (((n) + (d) - 1) / (d))
#define MEM_LEN(l)  (ROUNDUP((l), BMAP_BPW) * BMAP_WSIZE)


/*
 * Chunk flags.
 */
#define BMAP_CSET_DIRTY(cp) (cp)->c_flags |= BMAP_CDIRTY
#define BMAP_CDIRTY 0x00000001 /* the chunk is dirty */


/*
 * Macros on chunk flags.
 */
#define BMAP_CIS_DIRTY(cp)  ((cp)->c_flags & BMAP_CDIRTY)


/*
 * When loading a bitmap chunk, if it is new set the bitmap
 * can be set according to the initial value of bits.
 * Otherwise, it should be loaded from the file.
 */
#define BMAP_NEW_CHUNK      1
#define BMAP_OLD_CHUNK      0

/*
 * Each chunk holds the followin information:
 *  - A flag showing the status of the chunk, like being ditry or not.
 *  - Its offset in bits from the beginning of the vector.
 *  - Its length in bits.
 *  - Its memory length in use in bytes.
 *  - The bitmap vector.
 *
 *  In addition to the above information, each chunk can be on two lists:
 *  one the hash list, the other LRU list.  The hash list is a MRU list,
 *  meaning the MRU entry is at the head of the list.
 *
 *  All the chunks are in the LRU list. When a chunk is needed and there is
 *  no more room for allocating chunks, the first entry of this list is
 *  reclaimed.
 */
typedef struct dbmap_chunk {
    TAILQ_ENTRY(dbmap_chunk) c_hash;
    TAILQ_ENTRY(dbmap_chunk) c_lru;
    uint_t c_flags;
    u_quad_t c_off;
    uint_t c_clen;
    uint_t c_mlen;
    uint_t *c_bmp;
} dbmap_chunk_t;


TAILQ_HEAD(dbmap_list, dbmap_chunk);
typedef struct dbmap_list dbmap_list_t;


typedef struct dbitmap {
    char *bm_fname;
    int bm_fd;
    uint_t bm_flags;
    u_quad_t bm_len; /* bitmap length */
    uint_t bm_cmax; /* maximum number of cached chunks */
    uint_t bm_ccur; /* current number of cached chunks */
    dbmap_list_t bm_hash[BMAP_HASH_SIZE]; /* MRU hash table */
    dbmap_list_t bm_lru; /* LRU list */
} dbitmap_t;

/*
 * Disk bitmap table.  Upon allocating a dbitmap, one slot
 * of this table will be used.
 */
static dbitmap_t dbitmap[BMAP_MAX];


/*
 * Each chunk holds the followin information:
 *  - Its offset in bits from the beginning of the vector.
 *  - Its length in bits.
 *  - Its memory length in use in bytes.
 *  - The bitmap vector.
 *
 *  In addition to the above information, each chunk can be on a list:
 *  one the hash list.  The hash list is a MRU list,  meaning that the
 *  MRU entry is at the head of the list.
 */
typedef struct bmap_chunk {
    TAILQ_ENTRY(bmap_chunk) c_hash;
    u_quad_t c_off;
    uint_t c_clen;
    uint_t c_mlen;
    uint_t *c_bmp;
} bmap_chunk_t;


TAILQ_HEAD(bmap_list, bmap_chunk);
typedef struct bmap_list bmap_list_t;


typedef struct bitmap {
    uint_t bm_flags;
    u_quad_t bm_len; /* bitmap length */
    uint_t bm_cmax; /* maximum number of cached chunks */
    uint_t bm_ccur; /* current number of cached chunks */
    bmap_list_t bm_hash[BMAP_HASH_SIZE]; /* MRU hash table */
} bitmap_t;


/*
 * Statistics gathering structure.
 */
typedef struct bitmap_stats {
    ulong_t bs_alloc_cnt;
    ulong_t bs_alloc_size;
    ulong_t bs_free_cnt;
    ulong_t bs_set_applied;
    ulong_t bs_unset_applied;
    ulong_t bs_cache_hit;
    ulong_t bs_cache_miss;
    ulong_t bs_chunk_new;
    ulong_t bs_chunk_flush;
    ulong_t bs_chunk_reclaim;
    u_quad_t bs_get;
    u_quad_t bs_get_bits;
    u_quad_t bs_set;
    u_quad_t bs_set_bits;
    u_quad_t bs_unset;
    u_quad_t bs_unset_bits;
} bitmap_stats_t;


/*
 * Disk bitmap table.  Upon allocating a bitmap, one slot
 * of this table will be used.
 */
static bitmap_t bitmap[BMAP_MAX];


/*
 * Global instance of statistics variable.
 */
bitmap_stats_t bitmap_stats;


/*
 * bmd2bmp
 *
 * Convert bitmap descriptor to bitmap pointer.
 */
static bitmap_t *
bmd2bmp(int bmd)
{
    if (bmd < 0 || bmd >= BMAP_MAX)
        return (NULL);

    return (&bitmap[bmd]);
}


/*
 * bmd_alloc
 *
 * Allocate a bitmap descriptor.  Sets the INUSE flag of the slot.
 */
static int
bmd_alloc(void)
{
    int i;
    bitmap_t *bmp;

    bmp = bitmap;
    for (i = 0; i < BMAP_MAX; bmp++, i++)
        if (!BMAP_IS_INUSE(bmp)) {
            BMAP_SET_FLAGS(bmp, BMAP_INUSE);
            return (i);
        }

    return (-1);
}


/*
 * bmd_free
 *
 * Free a bitmap descriptor.  Clears the INUSE flag of the slot.
 */
static void
bmd_free(int bmd)
{
    bitmap_t *bmp;

    bmp = bmd2bmp(bmd);
    if (bmp)
        BMAP_UNSET_FLAGS(bmp, BMAP_INUSE);
}


/*
 * bmp_set
 *
 * Generic function to set bit in a chunk.  This can set or unset the
 * specified bit.
 */
static inline int
bmp_set(bmap_chunk_t *cp, u_quad_t bn, uint_t *vp)
{
    int rv;
    uint_t mask;
    uint_t *ip;
    uint_t v;

    bn -= cp->c_off;
    if (bn < cp->c_clen) {
        mask = 1 <<(bn & BMAP_BPW_MASK);
        ip = &cp->c_bmp[bn >> BMAP_BPW_SHIFT];
        v = (*vp <<(bn & BMAP_BPW_MASK)) & mask;
        *ip = (*ip & ~mask) | v;
        rv = 0;
    } else
        rv = -ERANGE;

    return (rv);
}


/*
 * bmp_get
 *
 * Generic function to get bit in a chunk.
 */
static inline int
bmp_get(bmap_chunk_t *cp, u_quad_t bn)
{
    int rv;
    uint_t bit;

    bn -= cp->c_off;
    if (bn < cp->c_clen) {
        bit = 1 <<(bn & BMAP_BPW_MASK);
        rv = (cp->c_bmp[bn >> BMAP_BPW_SHIFT] & bit) != 0;
    } else
        rv = -ERANGE;

    return (rv);
}


/*
 * bm_chuck_setup
 *
 * Set up the properties of the new chunk and position it in the hash list.
 */
static bmap_chunk_t *
bm_chunk_setup(bitmap_t *bmp, bmap_chunk_t *cp, u_quad_t bn)
{
    int h;
    u_quad_t off, l;
    uint_t cl, ml;
    bmap_list_t *hp;

    off = BMAP_CHUNK_OFF(bn);
    l = bmp->bm_len - off;
    if (l >= BMAP_CHUNK_BITS) {
        cl = BMAP_CHUNK_BITS;
        ml = BMAP_CHUNK_BYTES;
    } else {
        cl = l;
        ml = MEM_LEN(l);
    }

    if (BMAP_IS_INIT_ONES(bmp))
        (void) memset(cp->c_bmp, 0xff, ml);
    else
        (void) memset(cp->c_bmp, 0x00, ml);

    h = HASH(bn);
    hp = &bmp->bm_hash[h];

    TAILQ_INSERT_HEAD(hp, cp, c_hash);
    cp->c_off = off;
    cp->c_clen = cl;
    cp->c_mlen = ml;
    return (cp);
}


/*
 * bm_chunk_new
 *
 * Create a new chunk and keep track of memory used.
 */
static bmap_chunk_t *
bm_chunk_new(bitmap_t *bmp, u_quad_t bn)
{
    bmap_chunk_t *cp;

    bitmap_stats.bs_chunk_new++;

    cp = ndmp_malloc(sizeof (bmap_chunk_t));
    if (cp) {
        cp->c_bmp = ndmp_malloc(sizeof (uint_t) * BMAP_CHUNK_WORDS);
        if (!cp->c_bmp) {
            free(cp);
            cp = NULL;
        } else {
            (void) bm_chunk_setup(bmp, cp, bn);
            bmp->bm_ccur++;
        }
    }

    return (cp);
}


/*
 * bm_chunk_alloc
 *
 * Allocate a chunk and return it.  If the cache for the chunks is not
 * fully used, a new chunk is created.
 */
static bmap_chunk_t *
bm_chunk_alloc(bitmap_t *bmp, u_quad_t bn)
{
    bmap_chunk_t *cp;

    if (bmp->bm_ccur < bmp->bm_cmax)
        cp = bm_chunk_new(bmp, bn);
    else
        cp = NULL;

    return (cp);
}


/*
 * hash_free
 *
 * Free all chunks on the hash list.
 */
void
hash_free(bmap_list_t *hp)
{
    bmap_chunk_t *cp;

    if (!hp)
        return;

    while (!TAILQ_EMPTY(hp)) {
        cp = TAILQ_FIRST(hp);
        TAILQ_REMOVE(hp, cp, c_hash);
        free(cp->c_bmp);
        free(cp);
    }
}


/*
 * bm_chunks_free
 *
 * Release the memory allocated for the chunks.
 */
static void
bm_chunks_free(bmap_list_t *hp)
{
    int i;

    for (i = 0; i < BMAP_HASH_SIZE; hp++, i++)
        hash_free(hp);
}


/*
 * bm_chunk_repositions
 *
 * Re-position the chunk in the MRU hash table.
 */
static void
bm_chunk_reposition(bitmap_t *bmp, bmap_list_t *hp, bmap_chunk_t *cp)
{
    if (!bmp || !hp || !cp)
        return;

    if (TAILQ_FIRST(hp) != cp) {
        TAILQ_REMOVE(hp, cp, c_hash);
        TAILQ_INSERT_HEAD(hp, cp, c_hash);
    }
}


/*
 * bm_chunk_find
 *
 * Find and return the chunks which holds the specified bit. Allocate
 * the chunk if necessary and re-position it in the hash table lists.
 */
static bmap_chunk_t *
bm_chunk_find(bitmap_t *bmp, u_quad_t bn)
{
    int h;
    bmap_chunk_t *cp;
    bmap_list_t *hp;

    if (!bmp)
        return (NULL);

    h = HASH(bn);
    hp = &bmp->bm_hash[h];
    TAILQ_FOREACH(cp, hp, c_hash) {
        if (bn >= cp->c_off && bn < (cp->c_off + cp->c_clen)) {
            bitmap_stats.bs_cache_hit++;

            bm_chunk_reposition(bmp, hp, cp);
            return (cp);
        }
    }

    bitmap_stats.bs_cache_miss++;

    return (bm_chunk_alloc(bmp, bn));
}


/*
 * bmp_setval
 *
 * Set a range of bits in the bitmap specified by the vector.
 */
static int
bmp_setval(bitmap_t *bmp, bm_iovec_t *vp)
{
    int rv;
    u_quad_t cl;
    u_quad_t bn;
    u_quad_t max;
    bmap_chunk_t *cp;

    bn = vp->bmv_base;
    max = bn + vp->bmv_len;
    if (bn >= bmp->bm_len || max > bmp->bm_len)
        return (-EINVAL);

    if (*vp->bmv_val) {
        bitmap_stats.bs_set++;
        bitmap_stats.bs_set_bits += vp->bmv_len;
    } else {
        bitmap_stats.bs_unset++;
        bitmap_stats.bs_unset_bits += vp->bmv_len;
    }

    do {
        cp = bm_chunk_find(bmp, bn);
        if (!cp)
            return (-ERANGE);

        for (cl = cp->c_off + cp->c_clen; bn < cl && bn < max; bn++) {
            rv = bmp_set(cp, bn, vp->bmv_val);
            if (rv != 0)
                return (rv);
        }
    } while (bn < max);

    return (0);
}


/*
 * bmp_getval
 *
 * Get a range of bits in the bitmap specified by the vector.
 */
static int
bmp_getval(bitmap_t *bmp, bm_iovec_t *vp)
{
    uint_t cnt;
    uint_t *ip;
    int rv;
    u_quad_t cl;
    u_quad_t bn;
    u_quad_t max;
    bmap_chunk_t *cp;

    bn = vp->bmv_base;
    max = bn + vp->bmv_len;
    if (bn >= bmp->bm_len || max > bmp->bm_len)
        return (-EINVAL);

    bitmap_stats.bs_get++;
    bitmap_stats.bs_get_bits += 1;

    cnt = 0;
    ip = vp->bmv_val;
    *ip = 0;
    do {
        cp = bm_chunk_find(bmp, bn);
        if (!cp)
            return (-ERANGE);

        for (cl = cp->c_off + cp->c_clen; bn < cl && bn < max; bn++) {
            rv = bmp_get(cp, bn);
            if (rv < 0)
                return (rv);

            *ip |= rv << cnt;
            if (++cnt >= BMAP_BPW) {
                *++ip = 0;
                cnt = 0;
            }
        }
    } while (bn < max);

    return (0);
}


/*
 * hash_init
 *
 * Initialize the hash table lists head.
 */
static void
hash_init(bmap_list_t *hp)
{
    int i;

    for (i = 0; i < BMAP_HASH_SIZE; hp++, i++) {
        TAILQ_INIT(hp);
    }
}


/*
 * bm_alloc
 *
 * Allocate a bit map and return a handle to it.
 *
 * The hash table list are empty at this point. They are allocated
 * on demand.
 */
int
bm_alloc(u_quad_t len, int set)
{
    int bmd;
    bitmap_t *bmp;

    if (len == 0)
        return (-1);

    bmd = bmd_alloc();
    if (bmd < 0)
        return (bmd);

    bmp = bmd2bmp(bmd);
    bitmap_stats.bs_alloc_cnt++;
    bitmap_stats.bs_alloc_size += len;

    if (set)
        BMAP_SET_FLAGS(bmp, BMAP_BINIT_ONES);
    else
        BMAP_UNSET_FLAGS(bmp, BMAP_BINIT_ONES);
    bmp->bm_len = len;
    bmp->bm_ccur = 0;
    bmp->bm_cmax = BMAP_CHUNK_MAX;
    hash_init(bmp->bm_hash);

    return (bmd);
}


/*
 * bm_free
 *
 * Free memory allocated for the bitmap.
 */
int
bm_free(int bmd)
{
    int rv;
    bitmap_t *bmp;

    bmp = bmd2bmp(bmd);
    if (bmp && BMAP_IS_INUSE(bmp)) {
        bitmap_stats.bs_free_cnt++;

        bm_chunks_free(bmp->bm_hash);
        bmd_free(bmd);
        rv = 0;
    } else
        rv = -1;

    return (rv);
}


/*
 * bm_getiov
 *
 * Get bits specified by the array of vectors.
 */
int
bm_getiov(int bmd, bm_io_t *iop)
{
    int i;
    int rv;
    bm_iovec_t *vp;
    bitmap_t *bmp;

    if (!iop)
        rv = -EINVAL;
    else if (!(bmp = bmd2bmp(bmd)))
        rv = -EINVAL;
    else if (iop->bmio_iovcnt <= 0)
        rv = -EINVAL;
    else {
        rv = 0;
        vp = iop->bmio_iov;
        for (i = 0; i < iop->bmio_iovcnt; vp++, i++) {
            if (!vp)
                return (-EINVAL);
            rv |= bmp_getval(bmp, vp);
        }
    }

    return (rv);
}


/*
 * bm_setiov
 *
 * Set bits specified by the array of vectors.
 */
int
bm_setiov(int bmd, bm_io_t *iop)
{
    int i;
    int rv;
    bm_iovec_t *vp;
    bitmap_t *bmp;

    if (!iop)
        rv = -EINVAL;
    else if (!(bmp = bmd2bmp(bmd)))
        rv = -EINVAL;
    else if (iop->bmio_iovcnt <= 0)
        rv = -EINVAL;
    else if (!iop->bmio_iov)
        rv = -EINVAL;
    else {
        rv = 0;
        vp = iop->bmio_iov;
        for (i = 0; i < iop->bmio_iovcnt; vp++, i++)
            rv |= bmp_setval(bmp, vp);
    }

    return (rv);
}


/*
 * bmd2dbmp
 *
 * Convert bitmap descriptor to bitmap pointer.
 */
static dbitmap_t *
bmd2dbmp(int bmd)
{
    if (bmd < 0 || bmd >= BMAP_MAX)
        return (NULL);

    return (&dbitmap[bmd]);
}


/*
 * dbmp2bmd
 *
 * Convert bitmap pointer to bitmap descriptor.
 */
static int
dbmp2bmd(dbitmap_t *bmp)
{
    int bmd;

    bmd = bmp - dbitmap;
    if (bmd < 0 || bmd >= BMAP_MAX)
        bmd = -1;

    return (bmd);
}

/*
 * dbmd_alloc
 *
 * Allocate a bitmap descriptor.
 * Sets the INUSE flag of the slot.
 */
static int
dbmd_alloc(void)
{
    int i;
    dbitmap_t *bmp;

    bmp = dbitmap;
    for (i = 0; i < BMAP_MAX; bmp++, i++)
        if (!BMAP_IS_INUSE(bmp)) {
            BMAP_SET_FLAGS(bmp, BMAP_INUSE);
            return (i);
        }

    return (-1);
}


/*
 * dbmd_free
 *
 * Free a bitmap descriptor.
 * Clears the INUSE flag of the slot.
 */
static void
dbmd_free(int bmd)
{
    dbitmap_t *bmp;

    bmp = bmd2dbmp(bmd);
    if (bmp)
        BMAP_UNSET_FLAGS(bmp, BMAP_INUSE);
}


/*
 * dbmp_set
 *
 * Generic function to set bit in a chunk.  This can
 * set or unset the specified bit.
 */
static inline int
dbmp_set(dbmap_chunk_t *cp, u_quad_t bn, uint_t *vp)
{
    int rv;
    uint_t mask;
    uint_t *ip;
    uint_t v;

    bn -= cp->c_off;
    if (bn < cp->c_clen) {
        mask = 1 <<(bn & BMAP_BPW_MASK);
        ip = &cp->c_bmp[bn >> BMAP_BPW_SHIFT];
        v = (*vp <<(bn & BMAP_BPW_MASK)) & mask;
        *ip = (*ip & ~mask) | v;
        BMAP_CSET_DIRTY(cp);
        rv = 0;
    } else
        rv = -ERANGE;

    return (rv);
}


/*
 * dbmp_getlen
 *
 * Get length of the bitmap.
 */
static u_quad_t
dbmp_getlen(dbitmap_t *bmp)
{
    return (bmp ? bmp->bm_len : 0LL);
}


/*
 * dbmp_get
 *
 * Generic function to get bit in a chunk.
 */
static inline int
dbmp_get(dbmap_chunk_t *cp, u_quad_t bn)
{
    int rv;
    uint_t bit;

    bn -= cp->c_off;
    if (bn < cp->c_clen) {
        bit = 1 <<(bn & BMAP_BPW_MASK);
        rv = (cp->c_bmp[bn >> BMAP_BPW_SHIFT] & bit) != 0;
    } else
        rv = -ERANGE;

    return (rv);
}


/*
 * dbm_chunk_seek
 *
 * Seek in the file where the chunk is saved or should be saved.
 */
static int
dbm_chunk_seek(dbitmap_t *bmp, u_quad_t bn)
{
    int rv;
    off_t off;

    if (!bmp)
        rv = -1;
    else {
        off = BMAP_CHUNK_NO(bn) * BMAP_CHUNK_BYTES;
        rv = (lseek(bmp->bm_fd, off, SEEK_SET) != off) ? -1 : 0;
    }

    return (rv);
}


/*
 * dbm_chunk_flush
 *
 * Save a chunk to file.
 */
static int
dbm_chunk_flush(dbitmap_t *bmp, dbmap_chunk_t *cp)
{
    int rv;

    bitmap_stats.bs_chunk_flush++;
    if (!bmp || !cp)
        rv = -1;
    else if (dbm_chunk_seek(bmp, cp->c_off) != 0)
        rv = -1;
    else if (write(bmp->bm_fd, cp->c_bmp, cp->c_mlen) != cp->c_mlen)
        rv = -1;
    else
        rv = 0;

    return (rv);
}


/*
 * dbm_chunk_load
 *
 * Load a chunk from a file.  If the chunk is a new one,
 * instead of reading from the disk, the memory for the
 * chunk is set to either all zeros or to all ones.
 * Otherwise, if the chunk is not a new one, it's read
 * from the disk.
 *
 * The new chunk is positioned in the LRU and hash table
 * after its data is ready.
 */
static dbmap_chunk_t *
dbm_chunk_load(dbitmap_t *bmp, dbmap_chunk_t *cp, u_quad_t bn, int new)
{
    int h;
    u_quad_t off, l;
    uint_t cl, ml;
    dbmap_list_t *hp;

    off = BMAP_CHUNK_OFF(bn);
    l = bmp->bm_len - off;
    if (l >= BMAP_CHUNK_BITS) {
        cl = BMAP_CHUNK_BITS;
        ml = BMAP_CHUNK_BYTES;
    } else {
        cl = l;
        ml = MEM_LEN(l);
    }

    if (new == BMAP_NEW_CHUNK) {
        if (BMAP_IS_INIT_ONES(bmp))
            (void) memset(cp->c_bmp, 0xff, ml);
        else
            (void) memset(cp->c_bmp, 0x00, ml);
    } else { /* BMAP_OLD_CHUNK */
        if (dbm_chunk_seek(bmp, bn) != 0)
            cp = NULL;
        else if (read(bmp->bm_fd, cp->c_bmp, ml) != ml)
            cp = NULL;
    }

    if (cp) {
        TAILQ_INSERT_TAIL(&bmp->bm_lru, cp, c_lru);
        h = HASH(bn);
        hp = &bmp->bm_hash[h];
        TAILQ_INSERT_HEAD(hp, cp, c_hash);
        cp->c_flags = 0;
        cp->c_off = off;
        cp->c_clen = cl;
        cp->c_mlen = ml;
    }

    return (cp);
}


/*
 * dbm_chunk_new
 *
 * Create a new chunk and keep track of memory used.
 */
static dbmap_chunk_t *
dbm_chunk_new(dbitmap_t *bmp, u_quad_t bn)
{
    dbmap_chunk_t *cp;

    bitmap_stats.bs_chunk_new++;
    cp = ndmp_malloc(sizeof (dbmap_chunk_t));
    if (cp) {
        cp->c_bmp = ndmp_malloc(sizeof (uint_t) * BMAP_CHUNK_WORDS);
        if (!cp->c_bmp) {
            free(cp);
            cp = NULL;
        } else if (!dbm_chunk_load(bmp, cp, bn, BMAP_NEW_CHUNK)) {
            free(cp->c_bmp);
            free(cp);
            cp = NULL;
        } else
            bmp->bm_ccur++;
    }

    return (cp);
}


/*
 * dbm_chunk_alloc
 *
 * Allocate a chunk and return it.  If the cache for the
 * chunks is not fully used, a new chunk is created.
 * Otherwise, the first chunk from the LRU list is reclaimed,
 * loaded and returned.
 */
static dbmap_chunk_t *
dbm_chunk_alloc(dbitmap_t *bmp, u_quad_t bn)
{
    int h;
    dbmap_list_t *hp;
    dbmap_chunk_t *cp;

    if (bmp->bm_ccur < bmp->bm_cmax)
        return (dbm_chunk_new(bmp, bn));

    bitmap_stats.bs_chunk_reclaim++;

    cp = TAILQ_FIRST(&bmp->bm_lru);
    if (BMAP_CIS_DIRTY(cp))
        (void) dbm_chunk_flush(bmp, cp);

    TAILQ_REMOVE(&bmp->bm_lru, cp, c_lru);
    h = HASH(cp->c_off);
    hp = &bmp->bm_hash[h];
    TAILQ_REMOVE(hp, cp, c_hash);
    return (dbm_chunk_load(bmp, cp, bn, BMAP_OLD_CHUNK));
}


/*
 * dbm_chunks_free
 *
 * Release the memory allocated for the chunks.
 */
static void
dbm_chunks_free(dbitmap_t *bmp)
{
    dbmap_list_t *headp;
    dbmap_chunk_t *cp;

    if (!bmp)
        return;

    headp = &bmp->bm_lru;
    if (!headp)
        return;

    while (!TAILQ_EMPTY(headp)) {
        cp = TAILQ_FIRST(headp);
        TAILQ_REMOVE(headp, cp, c_lru);
        free(cp->c_bmp);
        free(cp);
    }
}


/*
 * dbm_chunk_reposition
 *
 * Re-position the chunk in the LRU and the hash table.
 */
static void
dbm_chunk_reposition(dbitmap_t *bmp, dbmap_list_t *hp, dbmap_chunk_t *cp)
{
    if (bmp && hp && cp) {
        TAILQ_REMOVE(&bmp->bm_lru, cp, c_lru);
        TAILQ_INSERT_TAIL(&bmp->bm_lru, cp, c_lru);
        if (TAILQ_FIRST(hp) != cp) {
            TAILQ_REMOVE(hp, cp, c_hash);
            TAILQ_INSERT_HEAD(hp, cp, c_hash);
        }
    }
}


/*
 * dbm_chunk_find
 *
 * Find and return the chunks which holds the specified bit.
 * Allocate the chunk if necessary and re-position it in the
 * LRU and hash table lists.
 */
static dbmap_chunk_t *
dbm_chunk_find(dbitmap_t *bmp, u_quad_t bn)
{
    int h;
    dbmap_chunk_t *cp;
    dbmap_list_t *hp;

    if (!bmp)
        return (NULL);

    h = HASH(bn);
    hp = &bmp->bm_hash[h];
    TAILQ_FOREACH(cp, hp, c_hash) {
        if (bn >= cp->c_off && bn < (cp->c_off + cp->c_clen)) {
            bitmap_stats.bs_cache_hit++;

            dbm_chunk_reposition(bmp, hp, cp);
            return (cp);
        }
    }

    bitmap_stats.bs_cache_miss++;

    return (dbm_chunk_alloc(bmp, bn));
}


/*
 * dbmp_setval
 *
 * Set a range of bits in the bitmap specified by the
 * vector.
 */
static int
dbmp_setval(dbitmap_t *bmp, bm_iovec_t *vp)
{
    int rv;
    u_quad_t cl;
    u_quad_t bn;
    u_quad_t max;
    dbmap_chunk_t *cp;

    bn = vp->bmv_base;
    max = bn + vp->bmv_len;
    if (bn >= bmp->bm_len || max > bmp->bm_len)
        return (-EINVAL);

    if (*vp->bmv_val) {
        bitmap_stats.bs_set++;
        bitmap_stats.bs_set_bits += vp->bmv_len;
    } else {
        bitmap_stats.bs_unset++;
        bitmap_stats.bs_unset_bits += vp->bmv_len;
    }

    do {
        cp = dbm_chunk_find(bmp, bn);
        if (!cp)
            return (-ERANGE);

        for (cl = cp->c_off + cp->c_clen; bn < cl && bn < max; bn++) {
            rv = dbmp_set(cp, bn, vp->bmv_val);
            if (rv != 0)
                return (rv);
        }
    } while (bn < max);

    return (0);
}


/*
 * dbmp_getval
 *
 * Get a range of bits in the bitmap specified by the
 * vector.
 */
static int
dbmp_getval(dbitmap_t *bmp, bm_iovec_t *vp)
{
    uint_t cnt;
    uint_t *ip;
    int rv;
    u_quad_t cl;
    u_quad_t bn;
    u_quad_t max;
    dbmap_chunk_t *cp;

    bn = vp->bmv_base;
    max = bn + vp->bmv_len;
    if (bn >= bmp->bm_len || max > bmp->bm_len)
        return (-EINVAL);

    bitmap_stats.bs_get++;
    bitmap_stats.bs_get_bits += 1;

    cnt = 0;
    ip = vp->bmv_val;
    *ip = 0;
    do {
        cp = dbm_chunk_find(bmp, bn);
        if (!cp)
            return (-ERANGE);

        for (cl = cp->c_off + cp->c_clen; bn < cl && bn < max; bn++) {
            rv = dbmp_get(cp, bn);
            if (rv < 0)
                return (rv);

            *ip |= rv << cnt;
            if (++cnt >= BMAP_BPW) {
                *++ip = 0;
                cnt = 0;
            }
        }
    } while (bn < max);

    return (0);
}


/*
 * dbyte_apply_ifset
 *
 * Apply the function on the set bits of the specified word.
 */
static int
dbyte_apply_ifset(dbitmap_t *bmp, u_quad_t off, uint_t b, int(*fp)(),
    void *arg)
{
    int bmd;
    int rv;
    u_quad_t l;

    rv = 0;
    l = dbmp_getlen(bmp);
    bmd = dbmp2bmd(bmp);
    for (; b && off < l; off++) {
        if (b & 1) {
            bitmap_stats.bs_set_applied++;

            if ((rv = (*fp)(bmd, off, arg)))
                break;
        }
        b >>= 1;
    }

    return (rv);
}


/*
 * dbm_chunk_apply_ifset
 *
 * Apply the function on the set bits of the specified chunk.
 */
static int
dbm_chunk_apply_ifset(dbitmap_t *bmp, dbmap_chunk_t *cp, int(*fp)(),
    void *arg)
{
    int rv;
    uint_t *bp;
    uint_t i, m;
    u_quad_t q;

    rv = 0;
    bp = cp->c_bmp;
    q = cp->c_off;
    m = cp->c_mlen / BMAP_WSIZE;
    for (i = 0; i < m; q += BMAP_BPW, bp++, i++)
        if (*bp) {
            rv = dbyte_apply_ifset(bmp, q, *bp, fp, arg);
            if (rv != 0)
                break;
        }

    return (rv);
}


/*
 * swfile_trunc
 *
 * Truncate the rest of the swap file.
 */
static int
swfile_trunc(int fd)
{
    int rv;
    off_t off;

    /*
     * Get the current offset and truncate whatever is
     * after this point.
     */
    rv = 0;
    if ((off = lseek(fd, 0, SEEK_CUR)) < 0)
        rv = -1;
    else if (ftruncate(fd, off) != 0)
        rv = -1;

    return (rv);
}


/*
 * swfile_init
 *
 * Initialize the swap file.  The necessary disk space is
 * reserved by writing to the swap file for swapping the
 * chunks in/out of the file.
 */
static int
swfile_init(int fd, u_quad_t len, int set)
{
    u_quad_t i, n;
    uint_t cl, ml;
    uint_t buf[BMAP_CHUNK_WORDS];

    (void) memset(buf, set ? 0xff : 0x00, BMAP_CHUNK_BYTES);
    n = len / BMAP_CHUNK_BITS;
    for (i = 0; i < n; i++)
        if (write(fd, buf, BMAP_CHUNK_BYTES) != BMAP_CHUNK_BYTES)
            return (-1);

    cl = (uint_t)(len % BMAP_CHUNK_BITS);
    ml = MEM_LEN(cl);
    if (write(fd, buf, ml) != ml)
        return (-1);

    return (swfile_trunc(fd));
}


/*
 * dbm_alloc
 *
 * Allocate a bit map and return a handle to it.
 *
 * The swap file is created if it does not exist.
 * The file is truncated if it exists and is larger
 * than needed amount.
 *
 * The hash table and LRU list are empty at this point.
 * They are allocated and/or loaded on-demand.
 */
int
dbm_alloc(char *fname, u_quad_t len, int set)
{
    int fd;
    int bmd;
    dbitmap_t *bmp;

    if (!fname || !*fname || !len)
        return (-1);

    /*
     * When allocating bitmap, make sure there is enough
     * disk space by allocating needed disk space, for
     * writing back the dirty chunks when swaping them out.
     */
    bmd = dbmd_alloc();
    if (bmd < 0)
        return (bmd);

    bmp = bmd2dbmp(bmd);
    if ((fd = open(fname, O_RDWR|O_CREAT, 0600)) < 0)
        bmd = -1;
    else if (swfile_init(fd, len, set) < 0) {
        bmd = -1;
        (void) close(fd);
        (void) unlink(fname);
        dbmd_free(bmd);
        bmd = -1;
    } else if (!(bmp->bm_fname = strdup(fname))) {
        (void) close(fd);
        (void) unlink(fname);
        dbmd_free(bmd);
        bmd = -1;
    } else {
        bitmap_stats.bs_alloc_cnt++;
        bitmap_stats.bs_alloc_size += len;

        bmp->bm_fd = fd;
        if (set)
            BMAP_SET_FLAGS(bmp, BMAP_BINIT_ONES);
        else
            BMAP_UNSET_FLAGS(bmp, BMAP_BINIT_ONES);
        bmp->bm_len = len;
        bmp->bm_ccur = 0;
        bmp->bm_cmax = BMAP_CHUNK_MAX;
        TAILQ_INIT(&bmp->bm_lru);
        hash_init((bmap_list_t *)bmp->bm_hash);
    }

    return (bmd);
}


/*
 * dbm_free
 *
 * Free memory allocated for the bitmap and remove its swap file.
 */
int
dbm_free(int bmd)
{
    int rv;
    dbitmap_t *bmp;

    bmp = bmd2dbmp(bmd);
    if (bmp && BMAP_IS_INUSE(bmp)) {
        bitmap_stats.bs_free_cnt++;

        dbm_chunks_free(bmp);
        (void) close(bmp->bm_fd);
        (void) unlink(bmp->bm_fname);
        free(bmp->bm_fname);
        dbmd_free(bmd);
        rv = 0;
    } else
        rv = -1;

    return (rv);
}


/*
 * dbm_getlen
 *
 * Return length of the bitmap.
 */
u_quad_t
dbm_getlen(int bmd)
{
    dbitmap_t *bmp;

    bmp = bmd2dbmp(bmd);
    return (dbmp_getlen(bmp));
}


/*
 * dbm_set
 *
 * Set a range of bits.
 */
int
dbm_set(int bmd, u_quad_t start, u_quad_t len, uint_t val)
{
    bm_io_t io;
    bm_iovec_t iov;

    iov.bmv_base = start;
    iov.bmv_len = len;
    iov.bmv_val = &val;
    io.bmio_iovcnt = 1;
    io.bmio_iov = &iov;

    return (dbm_setiov(bmd, &io));
}


/*
 * dbm_getiov
 *
 * Get bits specified by the array of vectors.
 */
int
dbm_getiov(int bmd, bm_io_t *iop)
{
    int i;
    int rv;
    bm_iovec_t *vp;
    dbitmap_t *bmp;

    if (!iop)
        rv = -EINVAL;
    else if (!(bmp = bmd2dbmp(bmd)))
        rv = -EINVAL;
    else if (iop->bmio_iovcnt <= 0)
        rv = -EINVAL;
    else {
        rv = 0;
        vp = iop->bmio_iov;
        for (i = 0; i < iop->bmio_iovcnt; vp++, i++) {
            if (!vp)
                return (-EINVAL);
            rv |= dbmp_getval(bmp, vp);
        }
    }

    return (rv);
}


/*
 * dbm_setiov
 *
 * Set bits specified by the array of vectors.
 */
int
dbm_setiov(int bmd, bm_io_t *iop)
{
    int i;
    int rv;
    bm_iovec_t *vp;
    dbitmap_t *bmp;

    if (!iop)
        rv = -EINVAL;
    else if (!(bmp = bmd2dbmp(bmd)))
        rv = -EINVAL;
    else if (iop->bmio_iovcnt <= 0)
        rv = -EINVAL;
    else if (!iop->bmio_iov)
        rv = -EINVAL;
    else {
        rv = 0;
        vp = iop->bmio_iov;
        for (i = 0; i < iop->bmio_iovcnt; vp++, i++)
            rv |= dbmp_setval(bmp, vp);
    }

    return (rv);
}


/*
 * dbm_apply_ifset
 *
 * Call the callback function for each set bit in the bitmap and
 * pass the 'arg' and bit number as its argument.
 */
int
dbm_apply_ifset(int bmd, int(*fp)(), void *arg)
{
    int rv;
    u_quad_t q;
    dbitmap_t *bmp;
    dbmap_chunk_t *cp;

    bmp = bmd2dbmp(bmd);
    if (!bmp || !fp)
        return (-EINVAL);

    rv = 0;
    for (q = 0; q < bmp->bm_len; q += BMAP_CHUNK_BITS) {
        cp = dbm_chunk_find(bmp, q);
        if (!cp) {
            rv = -ERANGE;
            break;
        }

        rv = dbm_chunk_apply_ifset(bmp, cp, fp, arg);
        if (rv != 0)
            break;
    }

    return (rv);
}


/*
 * bm_set
 *
 * Set a range of bits.
 */
int
bm_set(int bmd, u_quad_t start, u_quad_t len, uint_t val)
{
    bm_io_t io;
    bm_iovec_t iov;

    iov.bmv_base = start;
    iov.bmv_len = len;
    iov.bmv_val = &val;
    io.bmio_iovcnt = 1;
    io.bmio_iov = &iov;

    return (bm_setiov(bmd, &io));
}


/*
 * bm_get
 *
 * Get a range of bits.
 */
int
bm_get(int bmd, u_quad_t start, u_quad_t len, uint_t *buf)
{
    bm_io_t io;
    bm_iovec_t iov;

    iov.bmv_base = start;
    iov.bmv_len = len;
    iov.bmv_val = buf;
    io.bmio_iovcnt = 1;
    io.bmio_iov = &iov;

    return (bm_getiov(bmd, &io));
}


/*
 * bm_getone
 *
 * Get only one bit.
 */
int
bm_getone(int bmd, u_quad_t bitnum)
{
    uint_t i;

    if (bm_get(bmd, bitnum, 1, &i) == 0)
        return (i ? 1 : 0);

    return (0);
}


/*
 * dbm_get
 *
 * Get a range of bits.
 */
int
dbm_get(int bmd, u_quad_t start, u_quad_t len, uint_t *buf)
{
    bm_io_t io;
    bm_iovec_t iov;

    iov.bmv_base = start;
    iov.bmv_len = len;
    iov.bmv_val = buf;
    io.bmio_iovcnt = 1;
    io.bmio_iov = &iov;

    return (dbm_getiov(bmd, &io));
}


/*
 * dbm_getone
 *
 * Get only one bit.
 */
int
dbm_getone(int bmd, u_quad_t bitnum)
{
    uint_t i;

    if (dbm_get(bmd, bitnum, 1, &i) == 0)
        return (i ? 1 : 0);

    return (0);
}