backend/mech/crypto.c

	crypto.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 *  crypto.c
 *
 *  Copyright (c) 1997, by Sun Microsystems, Inc.
 *  All rights reserved.
 *
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

#include <sys/note.h>
#include "dh_gssapi.h"
#include "crypto.h"

/* Release the storage for a signature */
void
__free_signature(dh_signature_t sig)
{
    Free(sig->dh_signature_val);
    sig->dh_signature_val = NULL;
    sig->dh_signature_len = 0;
}

/* Release the storage for a gss_buffer */
void
__dh_release_buffer(gss_buffer_t b)
{
    Free(b->value);
    b->length = 0;
    b->value = NULL;
}

typedef struct cipher_entry {
    cipher_proc cipher; /* Routine to en/decrypt with */
    unsigned int pad;   /* Padding need for the routine */
} cipher_entry, *cipher_t;

typedef struct verifer_entry {
    verifier_proc msg;  /* Routine to calculate the check sum */
    unsigned int size;  /* Size of check sum */
    cipher_t signer;    /* Cipher entry to sign the check sum */
} verifier_entry, *verifier_t;

typedef struct QOP_entry {
    int export_level;   /* Not currentlyt used */
    verifier_t verifier;    /* Verifier entry to use for integrity */
} QOP_entry;

/*
 * Return the length produced by using cipher entry c given the supplied len
 */
static unsigned int
cipher_pad(cipher_t c, unsigned int len)
{
    unsigned int pad;

    pad = c ? c->pad : 1;

    return (((len + pad - 1)/pad)*pad);
}


/* EXPORT DELETE START */

/*
 * Des [en/de]crypt buffer, buf of length, len for each key provided using
 * an CBC initialization vector ivec.
 * If the mode is encrypt we will use the following pattern if the number
 * of keys is odd
 * encrypt(buf, k[0]), decrypt(buf, k[1]), encrypt(buf, k[2])
 *  decrypt(buf, k[4]) ... encrypt(buf, k[keynum - 1])
 * If we have an even number of keys and additional encryption will be
 * done with the first key, i.e., ecrypt(buf, k[0]);
 * In each [en/de]cription above we will used the passed in CBC initialization
 * vector. The new initialization vector will be the vector return from the
 * last encryption.
 *
 * In the decryption case we reverse the proccess. Note in this case
 * the return ivec will be from the first decryption.
 */

static int
__desN_crypt(des_block keys[], int keynum, char *buf, unsigned int len,
    unsigned int mode, char *ivec)
{
    /* Get the direction of ciphering */
    unsigned int m = mode & (DES_ENCRYPT | DES_DECRYPT);
    /* Get the remaining flags from mode */
    unsigned int flags = mode & ~(DES_ENCRYPT | DES_DECRYPT);
    des_block svec, dvec;
    int i, j, stat;

    /* Do we have at least one key */
    if (keynum < 1)
        return (DESERR_BADPARAM);

    /* Save the passed in ivec */
    memcpy(svec.c, ivec, sizeof (des_block));

    /* For  each key do the appropriate cipher */
    for (i = 0; i < keynum; i++) {
        j = (mode & DES_DECRYPT) ? keynum - 1 - i : i;
        stat = cbc_crypt(keys[j].c, buf, len, m | flags, ivec);
        if (mode & DES_DECRYPT && i == 0)
            memcpy(dvec.c, ivec, sizeof (des_block));

        if (DES_FAILED(stat))
            return (stat);

        m = (m == DES_ENCRYPT ? DES_DECRYPT : DES_ENCRYPT);

        if ((mode & DES_DECRYPT) || i != keynum - 1 || i%2)
            memcpy(ivec, svec.c, sizeof (des_block));
    }

    /*
     * If we have an even number of keys then do an extra round of
     * [en/de]cryption with the first key.
     */
    if (keynum % 2 == 0)
        stat = cbc_crypt(keys[0].c, buf, len, mode, ivec);

    /* If were decrypting ivec is set from first decryption */
    if (mode & DES_DECRYPT)
        memcpy(ivec, dvec.c, sizeof (des_block));

    return (stat);
}

/* EXPORT DELETE END */


/*
 * DesN crypt packaged for use as a cipher entry
 */
static OM_uint32
__dh_desN_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
{
    int stat = DESERR_BADPARAM;
/* EXPORT DELETE START */
    int encrypt_flag = (cipher_mode == ENCIPHER);
    unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
    des_block ivec;

    if (keys->dh_key_set_len < 1)
        return (DH_BADARG_FAILURE);

    /*
     * We all ways start of with ivec set to zeros. There is no
     * good way to maintain ivecs since packets could be out of sequence
     * duplicated or worst of all lost. Under these conditions the
     * higher level protocol would have to some how resync the ivecs
     * on both sides and start again. Theres no mechanism for this in
     * GSS.
     */
    memset(&ivec, 0, sizeof (ivec));

    /* Do the encryption/decryption */
    stat = __desN_crypt(keys->dh_key_set_val, keys->dh_key_set_len,
                (char *)buf->value, buf->length, mode, ivec.c);
/* EXPORT DELETE END */

    if (DES_FAILED(stat))
        return (DH_CIPHER_FAILURE);

    return (DH_SUCCESS);
}

/*
 * Package up plain des cbc crypt for use as a cipher entry.
 */
static OM_uint32
__dh_des_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
{
    int stat = DESERR_BADPARAM;
/* EXPORT DELETE START */
    int encrypt_flag = (cipher_mode == ENCIPHER);
    unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
    des_block ivec;

    if (keys->dh_key_set_len < 1)
        return (DH_BADARG_FAILURE);

    /*  Set the ivec to zeros and then cbc crypt the result */
    memset(&ivec, 0, sizeof (ivec));
    stat = cbc_crypt(keys->dh_key_set_val[0].c, (char *)buf->value,
            buf->length, mode, ivec.c);
/* EXPORT DELETE END */

    if (DES_FAILED(stat))
        return (DH_CIPHER_FAILURE);

    return (DH_SUCCESS);
}

/*
 * MD5_verifier: This is a verifier routine suitable for use in a
 * verifier entry. It calculates the MD5 check sum over an optional
 * msg and a token. It signs it using the supplied cipher_proc and stores
 * the result in signature.
 *
 * Note signature should already be allocated and be large enough to
 * hold the signature after its been encrypted. If keys is null, then
 * we will just return the unencrypted check sum.
 */
static OM_uint32
MD5_verifier(gss_buffer_t tok, /* The buffer to sign */
        gss_buffer_t msg, /* Optional buffer to include */
        cipher_proc signer, /* Routine to encrypt the integrity check */
        dh_key_set_t keys, /* Optiona keys to be used with the above */
        dh_signature_t signature /* The resulting MIC */)
{
    MD5_CTX md5_ctx;    /* MD5 context */
    gss_buffer_desc buf;    /* GSS buffer to hold keys for cipher routine */

    /* Initialize the MD5 context */
    MD5Init(&md5_ctx);
    /* If we have a message to digest, digest it */
    if (msg)
        MD5Update(&md5_ctx, (unsigned char *)msg->value, msg->length);
    /* Digest the supplied token */
    MD5Update(&md5_ctx, (unsigned char *)tok->value, tok->length);
    /* Finalize the sum. The MD5 context contains the digets */
    MD5Final(&md5_ctx);

    /* Copy the digest to the signature */
    memcpy(signature->dh_signature_val, (void *)md5_ctx.digest, 16);

    buf.length = signature->dh_signature_len;
    buf.value = signature->dh_signature_val;

    /* If we have keys encrypt it */
    if (keys != NULL)
        return (signer(&buf, keys, ENCIPHER));

    return (DH_SUCCESS);
}

/* Cipher table */
static
cipher_entry cipher_tab[] = {
    { NULL, 1},
    { __dh_desN_crypt, 8},
    { __dh_des_crypt, 8}
};


#define __NO_CRYPT  &cipher_tab[0]
#define __DES_N_CRYPT   &cipher_tab[1]
#define __DES_CRYPT &cipher_tab[2]

/* Verifier table */
static
verifier_entry verifier_tab[] = {
    { MD5_verifier, 16, __DES_N_CRYPT },
    { MD5_verifier, 16, __DES_CRYPT }
};

/* QOP table */
static
QOP_entry QOP_table[] = {
    { 0, &verifier_tab[0] },
    { 0, &verifier_tab[1] }
};

#define QOP_ENTRIES (sizeof (QOP_table) / sizeof (QOP_entry))

/*
 * __dh_is_valid_QOP: Return true if qop is valid entry into the QOP
 * table, else return false.
 */
bool_t
__dh_is_valid_QOP(dh_qop_t qop)
{
    bool_t is_valid = FALSE;

    is_valid = qop < QOP_ENTRIES;

    return (is_valid);
}

/*
 * __alloc_sig: Allocate a signature for a given QOP. This takes into
 * account the size of the signature after padding for the encryption
 * routine.
 */
OM_uint32
__alloc_sig(dh_qop_t qop, dh_signature_t sig)
{
    OM_uint32 stat = DH_VERIFIER_FAILURE;
    verifier_entry *v;

    /* Check that the QOP is valid */
    if (!__dh_is_valid_QOP(qop))
        return (DH_UNKNOWN_QOP);

    /* Get the verifier entry from the QOP entry */
    v = QOP_table[qop].verifier;

    /* Calulate the length needed for the signature */
    sig->dh_signature_len = cipher_pad(v->signer, v->size);

    /* Allocate the signature */
    sig->dh_signature_val = (void*)New(char, sig->dh_signature_len);
    if (sig->dh_signature_val == NULL) {
        sig->dh_signature_len = 0;
        return (DH_NOMEM_FAILURE);
    }

    stat = DH_SUCCESS;

    return (stat);
}

/*
 * __get_sig_size: Return the total size needed for a signature given a QOP.
 */
OM_uint32
__get_sig_size(dh_qop_t qop, unsigned int *size)
{
    /* Check for valid QOP */
    if (__dh_is_valid_QOP(qop)) {
        /* Get the verifier entry */
        verifier_t v = QOP_table[qop].verifier;

        /* Return the size include the padding needed for encryption */
        *size = v ? cipher_pad(v->signer, v->size) : 0;

        return (DH_SUCCESS);
    }
    *size = 0;

    return (DH_UNKNOWN_QOP);
}

/*
 * __mk_sig: Generate a signature using a given qop over a token of a
 * given length and an optional message. We use the supplied keys to
 * encrypt the check sum if they are available. The output is place
 * in a preallocate signature, that was allocated using __alloc_sig.
 */
OM_uint32
__mk_sig(dh_qop_t qop, /* The QOP to use */
    char *tok, /* The token to sign */
    long len, /* The tokens length */
    gss_buffer_t mesg,  /* An optional message to be included */
    dh_key_set_t keys, /* The optional encryption keys */
    dh_signature_t sig /* The resulting MIC */)
{
    OM_uint32 stat = DH_VERIFIER_FAILURE;


    verifier_entry *v;  /* Verifier entry */
    gss_buffer_desc buf;    /* Buffer to package tok */

    /* Make sure the QOP is valid */
    if (!__dh_is_valid_QOP(qop))
        return (DH_UNKNOWN_QOP);

    /* Grab the verifier entry for the qop */
    v = QOP_table[qop].verifier;

    /* Package the token for use in a verifier_proc */
    buf.length = len;
    buf.value = tok;

    /*
     * Calculate the signature using the supplied keys. If keys
     * is null, the the v->signer->cipher routine will not be called
     * and sig will not be encrypted.
     */
    stat = (*v->msg)(&buf, mesg, v->signer->cipher, keys, sig);

    return (stat);
}

/*
 * __verify_sig: Verify that the supplied signature, sig, is the same
 * as the token verifier
 */
OM_uint32
__verify_sig(dh_token_t token, /* The token to be verified */
        dh_qop_t qop, /* The QOP to use */
        dh_key_set_t keys, /* The context session keys */
        dh_signature_t sig /* The signature from the serialized token */)
{
    OM_uint32 stat = DH_VERIFIER_FAILURE;

    cipher_proc cipher; /* cipher routine to use */
    gss_buffer_desc buf;    /* Packaging for sig */

    /* Check the QOP */
    if (!__dh_is_valid_QOP(qop))
        return (DH_UNKNOWN_QOP);

    /* Package up the supplied signature */
    buf.length = sig->dh_signature_len;
    buf.value = sig->dh_signature_val;

    /* Get the cipher proc to use from the verifier entry for qop */
    cipher = QOP_table[qop].verifier->signer->cipher;

    /* Encrypt the check sum using the supplied set of keys */
    if ((stat = (*cipher)(&buf, keys, ENCIPHER)) != DH_SUCCESS)
        return (stat);

    /* Compare the signatures */
    if (__cmpsig(sig, &token->verifier))
        return (DH_SUCCESS);

    stat = DH_VERIFIER_MISMATCH;

    return (stat);
}

/*
 * __cmpsig: Return true if two signatures are the same, else false.
 */
bool_t
__cmpsig(dh_signature_t s1, dh_signature_t s2)
{
    return (s1->dh_signature_len == s2->dh_signature_len &&
        memcmp(s1->dh_signature_val,
        s2->dh_signature_val, s1->dh_signature_len) == 0);
}

/*
 * wrap_msg_body: Wrap the message pointed to be in into a
 * message pointed to by out that has ben padded out by pad bytes.
 *
 * The output message looks like:
 * out->length = total length of out->value including any padding
 * out->value points to memory as follows:
 * +------------+-------------------------+---------|
 * | in->length | in->value               | XDR PAD |
 * +------------+-------------------------+---------|
 *    4 bytes      in->length bytes         0 - 3
 */
static OM_uint32
wrap_msg_body(gss_buffer_t in, gss_buffer_t out)
{
    XDR xdrs;           /* xdrs to wrap with */
    unsigned int len, out_len;  /* length  */
    size_t size;

    out->length = 0;
    out->value = 0;

    /* Make sure the address of len points to a 32 bit word */
    len = (unsigned int)in->length;
    if (len != in->length)
        return (DH_ENCODE_FAILURE);

    size = ((in->length + sizeof (OM_uint32) + 3)/4) * 4;
    out_len = size;
    if (out_len != size)
        return (DH_ENCODE_FAILURE);

    /* Allocate the output buffer and set the length */
    if ((out->value = (void *)New(char, len)) == NULL)
        return (DH_NOMEM_FAILURE);
    out->length = out_len;


    /* Create xdr stream to wrap into */
    xdrmem_create(&xdrs, out->value, out->length, XDR_ENCODE);

    /* Wrap the bytes in value */
    if (!xdr_bytes(&xdrs, (char **)&in->value, &len, len)) {
        __dh_release_buffer(out);
        return (DH_ENCODE_FAILURE);
    }

    return (DH_SUCCESS);
}

/*
 * __QOPSeal: Wrap the input message placing the output in output given
 * a valid QOP. If confidentialiy is requested it is ignored. We can't
 * support privacy. The return flag will always be zero.
 */
OM_uint32
__QOPSeal(dh_qop_t qop, /* The QOP to use */
    gss_buffer_t input, /* The buffer to wrap */
    int conf_req, /* Do we want privacy ? */
    dh_key_set_t keys, /* The session keys */
    gss_buffer_t output, /* The wraped message */
    int *conf_ret /* Did we encrypt it? */)
{
_NOTE(ARGUNUSED(conf_req,keys))
    OM_uint32 stat = DH_CIPHER_FAILURE;

    *conf_ret = FALSE;  /* No encryption allowed */

    /* Check for valid QOP */
    if (!__dh_is_valid_QOP(qop))
        return (DH_UNKNOWN_QOP);

    /* Wrap the message */
    if ((stat = wrap_msg_body(input, output))
        != DH_SUCCESS)
        return (stat);

    return (stat);
}

/*
 * unwrap_msg_body: Unwrap the message, that was wrapped from above
 */
static OM_uint32
unwrap_msg_body(gss_buffer_t in, gss_buffer_t out)
{
    XDR xdrs;
    unsigned int len;   /* sizeof (len) == 32bits */

    /* Create an xdr stream to on wrap in */
    xdrmem_create(&xdrs, in->value, in->length, XDR_DECODE);

    /* Unwrap the input into out->value */
    if (!xdr_bytes(&xdrs, (char **)&out->value, &len, in->length))
        return (DH_DECODE_FAILURE);

    /* set the length */
    out->length = len;

    return (DH_SUCCESS);
}

/*
 * __QOPUnSeal: Unwrap the input message into output using the supplied QOP.
 * Note it is the callers responsibility to release the allocated output
 * buffer. If conf_req is true we return DH_CIPHER_FAILURE since we don't
 * support privacy.
 */
OM_uint32
__QOPUnSeal(dh_qop_t qop, /* The QOP to use */
        gss_buffer_t input, /* The message to unwrap */
        int conf_req, /* Is the message encrypted */
        dh_key_set_t keys, /* The session keys to decrypt if conf_req */
        gss_buffer_t output /* The unwraped message */)
{
_NOTE(ARGUNUSED(keys))
    OM_uint32 stat = DH_CIPHER_FAILURE;

    /* Check that the qop is valid */
    if (!__dh_is_valid_QOP(qop))
        return (DH_UNKNOWN_QOP);

    /* Set output to sane values */
    output->length = 0;
    output->value = NULL;

    /* Fail if this is privacy */
    if (conf_req)
        return (DH_CIPHER_FAILURE);

    /* Unwrap the input into the output, return the status */
    stat = unwrap_msg_body(input, output);

    return (stat);
}