crypto/modes/gcm.c

4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * CDDL HEADER START
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers *
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * The contents of this file are subject to the terms of the
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * Common Development and Distribution License (the "License").
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * You may not use this file except in compliance with the License.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers *
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * or http://www.opensolaris.org/os/licensing.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * See the License for the specific language governing permissions
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * and limitations under the License.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers *
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * When distributing Covered Code, include this CDDL HEADER in each
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * If applicable, add the following below this CDDL HEADER, with the
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * fields enclosed by brackets "[]" replaced with your own identifying
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * information: Portions Copyright [yyyy] [name of copyright owner]
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers *
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * CDDL HEADER END
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/*
7417cfdecea1902cef03c0d61a72df97d945925dKuriakose Kuruvilla * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#ifndef _KERNEL
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <strings.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <limits.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <assert.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <security/cryptoki.h>
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* _KERNEL */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <sys/types.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <sys/kmem.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <modes/modes.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <sys/crypto/common.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <sys/crypto/impl.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#include <sys/byteorder.h>
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#ifdef __amd64
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
95fddab55b9e310853e6cd5cd514291ae1c9016fDan OpenSolaris Anderson#ifdef _KERNEL
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#include <sys/cpuvar.h>     /* cpu_t, CPU */
7417cfdecea1902cef03c0d61a72df97d945925dKuriakose Kuruvilla#include <sys/x86_archext.h>    /* x86_featureset, X86FSET_*, CPUID_* */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#include <sys/disp.h>       /* kpreempt_disable(), kpreempt_enable */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson/* Workaround for no XMM kernel thread save/restore */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#define KPREEMPT_DISABLE    kpreempt_disable()
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#define KPREEMPT_ENABLE     kpreempt_enable()
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#else
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#include <sys/auxv.h>       /* getisax() */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#include <sys/auxv_386.h>   /* AV_386_PCLMULQDQ bit */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#define KPREEMPT_DISABLE
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#define KPREEMPT_ENABLE
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* _KERNEL */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Andersonextern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Andersonstatic int intel_pclmulqdq_instruction_present(void);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* __amd64 */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersstruct aes_block {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t a;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t b;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers};
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson/*
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * gcm_mul()
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Perform a carry-less multiplication (that is, use XOR instead of the
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * multiply operator) on *x_in and *y and place the result in *res.
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson *
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Byte swap the input (*x_in and *y) and the output (*res).
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson *
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Note: x_in, y, and res all point to 16-byte numbers (an array of two
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * 64-bit integers).
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson */
e8c016ef6bb7e825057460cea4d904e2b90eb424Mark Powersvoid
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#ifdef __amd64
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    if (intel_pclmulqdq_instruction_present()) {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        KPREEMPT_DISABLE;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        gcm_mul_pclmulqdq(x_in, y, res);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        KPREEMPT_ENABLE;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    } else
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* __amd64 */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        static const uint64_t R = 0xe100000000000000ULL;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        struct aes_block z = {0, 0};
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        struct aes_block v;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        uint64_t x;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        int i, j;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        v.a = ntohll(y[0]);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        v.b = ntohll(y[1]);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        for (j = 0; j < 2; j++) {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson            x = ntohll(x_in[j]);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson            for (i = 0; i < 64; i++, x <<= 1) {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                if (x & 0x8000000000000000ULL) {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    z.a ^= v.a;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    z.b ^= v.b;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                }
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                if (v.b & 1ULL) {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    v.b = (v.a << 63)|(v.b >> 1);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    v.a = (v.a >> 1) ^ R;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                } else {
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    v.b = (v.a << 63)|(v.b >> 1);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                    v.a = v.a >> 1;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson                }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        res[0] = htonll(z.a);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        res[1] = htonll(z.b);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#define GHASH(c, d, t) \
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson    gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson    (uint64_t *)(void *)(t));
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * is done in another function.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    crypto_data_t *out, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t remainder = length;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t need;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *datap = (uint8_t *)data;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *blockp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *lastp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void *iov_or_mp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    offset_t offset;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *out_data_1;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *out_data_2;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t out_data_1_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (length + ctx->gcm_remainder_len < block_size) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* accumulate bytes here and return */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bcopy(datap,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            length);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_remainder_len += length;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_copy_to = datap;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    lastp = (uint8_t *)ctx->gcm_cb;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (out != NULL)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        crypto_init_ptrs(out, &iov_or_mp, &offset);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    do {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* Unprocessed data from last call. */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (ctx->gcm_remainder_len > 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            need = block_size - ctx->gcm_remainder_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            if (need > remainder)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                return (CRYPTO_DATA_LEN_RANGE);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                [ctx->gcm_remainder_len], need);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            blockp = (uint8_t *)ctx->gcm_remainder;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            blockp = datap;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * Increment counter. Counter bits are confined
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * to the bottom 32 bits of the counter block.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = ntohll(ctx->gcm_cb[1] & counter_mask);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = htonll(counter + 1);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter &= counter_mask;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            (uint8_t *)ctx->gcm_tmp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        xor_block(blockp, (uint8_t *)ctx->gcm_tmp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        lastp = (uint8_t *)ctx->gcm_tmp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_processed_data_len += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (out == NULL) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            if (ctx->gcm_remainder_len > 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                bcopy(blockp, ctx->gcm_copy_to,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                    ctx->gcm_remainder_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                bcopy(blockp + ctx->gcm_remainder_len, datap,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                    need);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                &out_data_1_len, &out_data_2, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            /* copy block to where it belongs */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            if (out_data_1_len == block_size) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                copy_block(lastp, out_data_1);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                bcopy(lastp, out_data_1, out_data_1_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                if (out_data_2 != NULL) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                    bcopy(lastp + out_data_1_len,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                        out_data_2,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                        block_size - out_data_1_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            /* update offset */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            out->cd_offset += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* add ciphertext to the hash */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* Update pointer to next block of data to be processed. */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (ctx->gcm_remainder_len != 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            datap += need;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            ctx->gcm_remainder_len = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            datap += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        remainder = (size_t)&data[length] - (size_t)datap;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* Incomplete last block. */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (remainder > 0 && remainder < block_size) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            bcopy(datap, ctx->gcm_remainder, remainder);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            ctx->gcm_remainder_len = remainder;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            ctx->gcm_copy_to = datap;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            goto out;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_copy_to = NULL;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    } while (remainder > 0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersout:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/* ARGSUSED */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *ghash, *macp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int i, rv;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (out->cd_length <
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (CRYPTO_DATA_LEN_RANGE);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ghash = (uint8_t *)ctx->gcm_ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (ctx->gcm_remainder_len > 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        uint64_t counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * Here is where we deal with data that is not a
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * multiple of the block size.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * Increment counter.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = ntohll(ctx->gcm_cb[1] & counter_mask);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = htonll(counter + 1);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter &= counter_mask;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            (uint8_t *)ctx->gcm_tmp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        macp = (uint8_t *)ctx->gcm_remainder;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bzero(macp + ctx->gcm_remainder_len,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            block_size - ctx->gcm_remainder_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* XOR with counter block */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        for (i = 0; i < ctx->gcm_remainder_len; i++) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            macp[i] ^= tmpp[i];
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* add ciphertext to the hash */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        GHASH(ctx, macp, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_processed_data_len += ctx->gcm_remainder_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson    ctx->gcm_len_a_len_c[1] =
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        (uint8_t *)ctx->gcm_J0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    xor_block((uint8_t *)ctx->gcm_J0, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (ctx->gcm_remainder_len > 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (rv != CRYPTO_SUCCESS)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            return (rv);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    out->cd_offset += ctx->gcm_remainder_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ctx->gcm_remainder_len = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (rv != CRYPTO_SUCCESS)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (rv);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    out->cd_offset += ctx->gcm_tag_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * This will only deal with decrypting the last block of the input that
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * might not be a multiple of block length.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersstatic void
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *datap, *outp, *counterp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int i;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     * Increment counter.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     * Counter bits are confined to the bottom 32 bits
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    counter = ntohll(ctx->gcm_cb[1] & counter_mask);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    counter = htonll(counter + 1);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    counter &= counter_mask;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    datap = (uint8_t *)ctx->gcm_remainder;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    outp = &((ctx->gcm_pt_buf)[index]);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    counterp = (uint8_t *)ctx->gcm_tmp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* authentication tag */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    bzero((uint8_t *)ctx->gcm_tmp, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* add ciphertext to the hash */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* decrypt remaining ciphertext */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* XOR with counter block */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    for (i = 0; i < ctx->gcm_remainder_len; i++) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        outp[i] = datap[i] ^ counterp[i];
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/* ARGSUSED */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    crypto_data_t *out, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t new_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *new;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     * Copy contiguous ciphertext input blocks to plaintext buffer.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     * Ciphertext will be decrypted in the final.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (length > 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        new_len = ctx->gcm_pt_buf_len + length;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#ifdef _KERNEL
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        new = kmem_alloc(new_len, ctx->gcm_kmflag);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#else
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        new = malloc(new_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        free(ctx->gcm_pt_buf);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#endif
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (new == NULL)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            return (CRYPTO_HOST_MEMORY);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_pt_buf = new;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_pt_buf_len = new_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            length);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_processed_data_len += length;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ctx->gcm_remainder_len = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t pt_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t remainder;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *blockp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *cbp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int processed = 0, rv;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ghash = (uint8_t *)ctx->gcm_ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    blockp = ctx->gcm_pt_buf;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    remainder = pt_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    while (remainder > 0) {
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick        /* Incomplete last block */
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick        if (remainder < block_size) {
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            bcopy(blockp, ctx->gcm_remainder, remainder);
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            ctx->gcm_remainder_len = remainder;
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            /*
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick             * not expecting anymore ciphertext, just
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick             * compute plaintext for the remaining input
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick             */
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            gcm_decrypt_incomplete_block(ctx, block_size,
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick                processed, encrypt_block, xor_block);
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            ctx->gcm_remainder_len = 0;
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick            goto out;
553d52d4675c67a5de5a10ad30305be1a208e218Mark Fenwick        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* add ciphertext to the hash */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        GHASH(ctx, blockp, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * Increment counter.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         * Counter bits are confined to the bottom 32 bits
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers         */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = ntohll(ctx->gcm_cb[1] & counter_mask);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter = htonll(counter + 1);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        counter &= counter_mask;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        cbp = (uint8_t *)ctx->gcm_tmp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* XOR with ciphertext */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        xor_block(cbp, blockp);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        processed += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        blockp += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        remainder -= block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersout:
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson    ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        (uint8_t *)ctx->gcm_J0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    xor_block((uint8_t *)ctx->gcm_J0, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* compare the input authentication tag with what we calculated */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* They don't match */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (CRYPTO_INVALID_MAC);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (rv != CRYPTO_SUCCESS)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            return (rv);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        out->cd_offset += pt_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersstatic int
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_validate_args(CK_AES_GCM_PARAMS *gcm_param)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t tag_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     * Check the length of the authentication tag (in bits).
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers     */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    tag_len = gcm_param->ulTagBits;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    switch (tag_len) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 32:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 64:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 96:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 104:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 112:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 120:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    case 128:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        break;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    default:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (CRYPTO_MECHANISM_PARAM_INVALID);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (gcm_param->ulIvLen == 0)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (CRYPTO_MECHANISM_PARAM_INVALID);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersstatic void
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    gcm_ctx_t *ctx, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *cb;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ulong_t remainder = iv_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ulong_t processed = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *datap, *ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint64_t len_a_len_c[2];
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ghash = (uint8_t *)ctx->gcm_ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    cb = (uint8_t *)ctx->gcm_cb;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (iv_len == 12) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        bcopy(iv, cb, 12);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        cb[12] = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        cb[13] = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        cb[14] = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        cb[15] = 1;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* J0 will be used again in the final */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        copy_block(cb, (uint8_t *)ctx->gcm_J0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* GHASH the IV */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        do {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            if (remainder < block_size) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                bzero(cb, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                bcopy(&(iv[processed]), cb, remainder);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                datap = (uint8_t *)cb;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                remainder = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                datap = (uint8_t *)(&(iv[processed]));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                processed += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers                remainder -= block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            GHASH(ctx, datap, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        } while (remainder > 0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        len_a_len_c[0] = 0;
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        GHASH(ctx, len_a_len_c, ctx->gcm_J0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* J0 will be used again in the final */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers/*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * The following function is called at encrypt or decrypt init time
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers * for AES GCM mode.
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    unsigned char *auth_data, size_t auth_data_len, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    uint8_t *ghash, *datap, *authp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    size_t remainder, processed;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    /* encrypt zero block to get subkey H */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        (uint8_t *)ctx->gcm_H);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        copy_block, xor_block);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    authp = (uint8_t *)ctx->gcm_tmp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ghash = (uint8_t *)ctx->gcm_ghash;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    bzero(authp, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    bzero(ghash, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    processed = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    remainder = auth_data_len;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    do {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if (remainder < block_size) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            /*
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers             * There's not a block full of data, pad rest of
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers             * buffer with zero
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers             */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            bzero(authp, block_size);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            bcopy(&(auth_data[processed]), authp, remainder);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            datap = (uint8_t *)authp;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            remainder = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            datap = (uint8_t *)(&(auth_data[processed]));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            processed += block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            remainder -= block_size;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* add auth data to the hash */
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        GHASH(ctx, datap, ghash);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    } while (remainder > 0);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (CRYPTO_SUCCESS);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersint
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*copy_block)(uint8_t *, uint8_t *),
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    void (*xor_block)(uint8_t *, uint8_t *))
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    int rv;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    CK_AES_GCM_PARAMS *gcm_param;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (param != NULL) {
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        gcm_param = (CK_AES_GCM_PARAMS *)(void *)param;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        if ((rv = gcm_validate_args(gcm_param)) != 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers            return (rv);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        gcm_ctx->gcm_tag_len = gcm_param->ulTagBits;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        gcm_ctx->gcm_tag_len >>= 3;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        gcm_ctx->gcm_processed_data_len = 0;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        /* these values are in bits */
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        gcm_ctx->gcm_len_a_len_c[0]
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson            = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen));
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        rv = CRYPTO_SUCCESS;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        gcm_ctx->gcm_flags |= GCM_MODE;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    } else {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        rv = CRYPTO_MECHANISM_PARAM_INVALID;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        goto out;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        gcm_param->pAAD, gcm_param->ulAADLen, block_size,
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        encrypt_block, copy_block, xor_block) != 0) {
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        rv = CRYPTO_MECHANISM_PARAM_INVALID;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    }
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersout:
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (rv);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powersint
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powersgmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    void (*copy_block)(uint8_t *, uint8_t *),
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    void (*xor_block)(uint8_t *, uint8_t *))
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers{
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    int rv;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    CK_AES_GMAC_PARAMS *gmac_param;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    if (param != NULL) {
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS);
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        gcm_ctx->gcm_processed_data_len = 0;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        /* these values are in bits */
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson        gcm_ctx->gcm_len_a_len_c[0]
95014fbbfdc010ab9f3ed20db2154dc3322e9270Dan OpenSolaris Anderson            = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen));
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        rv = CRYPTO_SUCCESS;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        gcm_ctx->gcm_flags |= GMAC_MODE;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    } else {
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        rv = CRYPTO_MECHANISM_PARAM_INVALID;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        goto out;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    }
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN,
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        gmac_param->pAAD, gmac_param->ulAADLen, block_size,
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        encrypt_block, copy_block, xor_block) != 0) {
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        rv = CRYPTO_MECHANISM_PARAM_INVALID;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    }
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powersout:
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    return (rv);
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers}
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersvoid *
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_alloc_ctx(int kmflag)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    gcm_ctx_t *gcm_ctx;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#ifdef _KERNEL
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#else
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers#endif
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers        return (NULL);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    gcm_ctx->gcm_flags = GCM_MODE;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    return (gcm_ctx);
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powersvoid *
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powersgmac_alloc_ctx(int kmflag)
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers{
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    gcm_ctx_t *gcm_ctx;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers#ifdef _KERNEL
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers#else
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL)
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers#endif
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers        return (NULL);
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    gcm_ctx->gcm_flags = GMAC_MODE;
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers    return (gcm_ctx);
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers}
983a10335731bc55a0b7a37f195575fa109e30d4Mark Powers
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersvoid
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powersgcm_set_kmflag(gcm_ctx_t *ctx, int kmflag)
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers{
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers    ctx->gcm_kmflag = kmflag;
4d703b5c9cb1bc29ace6c53cb50b2fe766e6370fMark Powers}
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#ifdef __amd64
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson/*
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Return 1 if executing on Intel with PCLMULQDQ instructions,
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64).
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Cache the result, as the CPU can't change.
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson *
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson * Note: the userland version uses getisax().  The kernel version uses
7417cfdecea1902cef03c0d61a72df97d945925dKuriakose Kuruvilla * is_x86_featureset().
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Andersonstatic int
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Andersonintel_pclmulqdq_instruction_present(void)
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson{
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    static int  cached_result = -1;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    if (cached_result == -1) { /* first time */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#ifdef _KERNEL
7417cfdecea1902cef03c0d61a72df97d945925dKuriakose Kuruvilla        cached_result =
7417cfdecea1902cef03c0d61a72df97d945925dKuriakose Kuruvilla            is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#else
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        uint_t      ui = 0;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        (void) getisax(&ui, 1);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson        cached_result = (ui & AV_386_PCLMULQDQ) != 0;
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* _KERNEL */
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    }
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson    return (cached_result);
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson}
104d3bde5b4ac46904f144d3676110fc57a69603Dan OpenSolaris Anderson#endif  /* __amd64 */