io/sfxge/sfxge_hash.c

49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/*
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Copyright (c) 2008-2016 Solarflare Communications Inc.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * All rights reserved.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Redistribution and use in source and binary forms, with or without
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * modification, are permitted provided that the following conditions are met:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * 1. Redistributions of source code must retain the above copyright notice,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *    this list of conditions and the following disclaimer.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * 2. Redistributions in binary form must reproduce the above copyright notice,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *    this list of conditions and the following disclaimer in the documentation
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *    and/or other materials provided with the distribution.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The views and conclusions contained in the software and documentation are
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * those of the authors and should not be interpreted as representing official
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * policies, either expressed or implied, of the FreeBSD Project.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/param.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/int_limits.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/byteorder.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/random.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/types.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <sys/kmem.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include <netinet/in.h>
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include "sfxge.h"
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#include "efx.h"
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/*
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The largest amount of the data which the hash may be calculated over
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * is a 4-tuple of source/destination IPv6 addresses (2 x 16 bytes)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * and source/destination TCP port numbers (2 x 2 bytes), adding up to 40 bytes
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#define SFXGE_TOEPLITZ_IN_MAX \
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    (2 * (sizeof (struct in6_addr) + sizeof (in_port_t)))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#define SFXGE_TOEPLITZ_CACHE_SIZE (SFXGE_TOEPLITZ_IN_MAX * (UINT8_MAX + 1))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic uint32_t
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoretoeplitz_hash(const uint32_t *cache, const uint8_t *input,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    unsigned pos, unsigned datalen)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore{
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    uint32_t hash = 0;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    for (; datalen != 0; datalen--, pos++, input++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        hash ^= cache[pos * (UINT8_MAX + 1) + *input];
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    return (hash);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore}
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoreuint32_t
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_toeplitz_hash(sfxge_t *sp, unsigned int addr_size,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    uint8_t *src_addr, uint16_t src_port, uint8_t *dst_addr, uint16_t dst_port)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore{
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    uint32_t hash = 0;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    unsigned pos = 0;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    hash ^= toeplitz_hash(sp->s_toeplitz_cache, src_addr, pos, addr_size);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    pos += addr_size;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    hash ^= toeplitz_hash(sp->s_toeplitz_cache, dst_addr, pos, addr_size);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    pos += addr_size;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    if (src_port != 0 || dst_port != 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        hash ^= toeplitz_hash(sp->s_toeplitz_cache,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            (const uint8_t *)&src_port, pos, sizeof (src_port));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        pos += sizeof (src_port);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        hash ^= toeplitz_hash(sp->s_toeplitz_cache,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            (const uint8_t *)&dst_port, pos, sizeof (dst_port));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    return (hash);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore}
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/*
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The algorithm to calculate RSS Toeplitz hash is essentially as follows:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - Regard a Toeplitz key and an input as bit strings, with the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * most significant bit of the first byte being the first bit
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - Let's have a 32-bit window sliding over the Toeplitz key bit by bit
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - Let the initial value of the hash be zero
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - Then for every bit in the input that is set to 1, XOR the value of the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *   window at a given bit position into the resulting hash
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * First we note that since XOR is commutative and associative, the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * resulting hash is just a XOR of subhashes for every input bit:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *        H = H_0 XOR H_1 XOR ... XOR H_n               (1)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Then we note that every H_i is only dependent on the value of i and
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * the value of i'th bit of input, but not on any preceding or following
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * input bits.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Then we note that (1) holds also for any bit sequences,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * e.g. for bytes of input:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *       H = H_0_7 XOR H_8_15 XOR ... XOR H_(n-7)_n     (2)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * and every
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *       H_i_j = H_i XOR H_(i+1) ... XOR H_j.           (3)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * It naturally follows than H_i_(i+7) only depends on the value of the byte
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * and the position of the byte in the input.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Therefore we may pre-calculate the value of each byte sub-hash H_i_(i+7)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * for each possible byte value and each possible byte input position, and
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * then just assemble the hash of the packet byte-by-byte instead of
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * bit-by-bit.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The amount of memory required for such a cache is not prohibitive:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - we have at most 36 bytes of input, each holding 256 possible values
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - and the hash is 32-bit wide
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * - hence, we need only 36 * 256 * 4 = 36kBytes of cache.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The performance gain, at least on synthetic benchmarks, is significant:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * cache lookup is about 15 times faster than direct hash calculation
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoreconst uint32_t *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoretoeplitz_cache_init(const uint8_t *key)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore{
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    uint32_t *cache = kmem_alloc(SFXGE_TOEPLITZ_CACHE_SIZE *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        sizeof (uint32_t), KM_SLEEP);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    unsigned i;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    for (i = 0; i < SFXGE_TOEPLITZ_IN_MAX; i++, key++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        uint32_t key_bits[NBBY] = { 0 };
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        unsigned j;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        unsigned mask;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        unsigned byte;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#if defined(BE_IN32)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        key_bits[0] = BE_IN32(key);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#else
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        key_bits[0] = BE_32(*(uint32_t *)key);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#endif
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        for (j = 1, mask = 1 << (NBBY - 1); j < NBBY; j++, mask >>= 1) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            key_bits[j] = key_bits[j - 1] << 1;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            if ((key[sizeof (uint32_t)] & mask) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore                key_bits[j] |= 1;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        for (byte = 0; byte <= UINT8_MAX; byte++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            uint32_t res = 0;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            for (j = 0, mask = 1 << (NBBY - 1);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore                j < NBBY;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore                j++, mask >>= 1) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore                if (byte & mask)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore                    res ^= key_bits[j];
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            cache[i * (UINT8_MAX + 1) + byte] = res;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    return (cache);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore}
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoreint
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_toeplitz_hash_init(sfxge_t *sp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore{
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    int rc;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    uint8_t toeplitz_key[SFXGE_TOEPLITZ_KEY_LEN];
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    (void) random_get_pseudo_bytes(toeplitz_key, sizeof (toeplitz_key));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    if ((rc = efx_rx_scale_mode_set(sp->s_enp, EFX_RX_HASHALG_TOEPLITZ,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        (1 << EFX_RX_HASH_IPV4) | (1 << EFX_RX_HASH_TCPIPV4) |
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        (1 << EFX_RX_HASH_IPV6) | (1 << EFX_RX_HASH_TCPIPV6), B_TRUE)) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        return (rc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    if ((rc = efx_rx_scale_key_set(sp->s_enp, toeplitz_key,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        sizeof (toeplitz_key))) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        return (rc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    sp->s_toeplitz_cache = toeplitz_cache_init(toeplitz_key);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    return (0);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore}
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorevoid
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_toeplitz_hash_fini(sfxge_t *sp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore{
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    if (sp->s_toeplitz_cache != NULL) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        kmem_free((void *)sp->s_toeplitz_cache,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore            SFXGE_TOEPLITZ_CACHE_SIZE);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore        sp->s_toeplitz_cache = NULL;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore    }
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore}