libsmb/common/smb_crypt.c

da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw/*
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * CDDL HEADER START
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw *
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * The contents of this file are subject to the terms of the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Common Development and Distribution License (the "License").
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * You may not use this file except in compliance with the License.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw *
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * or http://www.opensolaris.org/os/licensing.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * See the License for the specific language governing permissions
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * and limitations under the License.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw *
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * When distributing Covered Code, include this CDDL HEADER in each
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * If applicable, add the following below this CDDL HEADER, with the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * fields enclosed by brackets "[]" replaced with your own identifying
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * information: Portions Copyright [yyyy] [name of copyright owner]
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw *
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * CDDL HEADER END
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw/*
3db3f65c6274eb042354801a308c8e9bc4994553amw * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Use is subject to license terms.
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <sys/md4.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <sys/types.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <string.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <security/cryptoki.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <security/pkcs11.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <cryptoutil.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#include <smbsrv/libsmb.h>
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
3db3f65c6274eb042354801a308c8e9bc4994553amwstatic void smb_initlmkey(unsigned char *keyin, unsigned char *keyout);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross/*
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross * randomize
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross *
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross * Randomize the contents of the specified buffer.
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rossvoid
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rossrandomize(char *data, unsigned len)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross{
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    char *p = data;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (pkcs11_get_random(data, len) == 0)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        return;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    /*
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * Implement a "fall back", because current callers
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * don't expect an error from this.  In practice,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * we never use this fall back.
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    while (len--) {
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        *p++ = (random() >> 24);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    }
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross}
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw/*
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * smb_auth_md4
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw *
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Compute an MD4 digest.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwint
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwsmb_auth_md4(unsigned char *result, unsigned char *input, int length)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw{
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    MD4_CTX md4_context;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    MD4Init(&md4_context);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    MD4Update(&md4_context, input, length);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    MD4Final(result, &md4_context);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    return (SMBAUTH_SUCCESS);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw}
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwint
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwsmb_auth_hmac_md5(unsigned char *data,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    int data_len,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    unsigned char *key,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    int key_len,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    unsigned char *digest)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw{
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_RV rv;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_MECHANISM mechanism;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_OBJECT_HANDLE hKey;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_SESSION_HANDLE hSession;
3db3f65c6274eb042354801a308c8e9bc4994553amw    CK_ULONG diglen = MD_DIGEST_LEN;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.mechanism = CKM_MD5_HMAC;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.pParameter = 0;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.ulParameterLen = 0;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        key, key_len, &hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    /* Initialize the digest operation in the session */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = C_SignInit(hSession, &mechanism, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = C_SignUpdate(hSession, (CK_BYTE_PTR)data, data_len);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = C_SignFinal(hSession, (CK_BYTE_PTR)digest, &diglen);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (diglen != MD_DIGEST_LEN) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    return (SMBAUTH_SUCCESS);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw}
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwint
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwsmb_auth_DES(unsigned char *Result, int ResultLen,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    unsigned char *Key, int KeyLen,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    unsigned char *Data, int DataLen)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw{
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_RV rv;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_MECHANISM mechanism;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_OBJECT_HANDLE hKey;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_SESSION_HANDLE hSession;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    CK_ULONG ciphertext_len;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    uchar_t des_key[8];
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    int error = 0;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    int K, D;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    int k, d;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    /*
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * Calculate proper number of iterations.
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * Known call cases include:
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     *   ResultLen=16, KeyLen=14, DataLen=8
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     *   ResultLen=24, KeyLen=21, DataLen=8
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     *   ResultLen=16, KeyLen=14, DataLen=16
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    K = KeyLen / 7;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    D = DataLen / 8;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if ((KeyLen % 7) || (DataLen % 8))
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        return (EINVAL);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (K == 0 || D == 0)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        return (EINVAL);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (ResultLen < (K * 8))
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        return (EINVAL);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    /*
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw     * Use SUNW convenience function to initialize the cryptoki
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw     * library, and open a session with a slot that supports
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw     * the mechanism we plan on using.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw     */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.mechanism = CKM_DES_ECB;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.pParameter = NULL;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    mechanism.ulParameterLen = 0;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    for (d = k = 0; k < K; k++, d++) {
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        /* Cycle the input again, as necessary. */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        if (d == D)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            d = 0;
3db3f65c6274eb042354801a308c8e9bc4994553amw        smb_initlmkey(&Key[k * 7], des_key);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw            des_key, 8, &hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw            error = 1;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw            goto exit_session;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        /* Initialize the encryption operation in the session */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        rv = C_EncryptInit(hSession, &mechanism, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        if (rv != CKR_OK) {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw            error = 1;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw            goto exit_encrypt;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        }
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        ciphertext_len = 8;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        /* Read in the data and encrypt this portion */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        rv = C_EncryptUpdate(hSession,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            (CK_BYTE_PTR)Data + (d * 8), 8,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            (CK_BYTE_PTR)Result + (k * 8),
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            &ciphertext_len);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        if (rv != CKR_OK) {
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            error = 1;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross            goto exit_encrypt;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        }
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    }
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    goto exit_session;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwexit_encrypt:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    (void) C_DestroyObject(hSession, hKey);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwexit_session:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    (void) C_CloseSession(hSession);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    if (error)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw        return (SMBAUTH_FAILURE);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw    return (SMBAUTH_SUCCESS);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw}
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw/*
3db3f65c6274eb042354801a308c8e9bc4994553amw * See "Netlogon Credential Computation" section of MS-NRPC document.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwstatic void
3db3f65c6274eb042354801a308c8e9bc4994553amwsmb_initlmkey(unsigned char *keyin, unsigned char *keyout)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw{
3db3f65c6274eb042354801a308c8e9bc4994553amw    int i;
3db3f65c6274eb042354801a308c8e9bc4994553amw
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[0] = keyin[0] >> 0x01;
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
3db3f65c6274eb042354801a308c8e9bc4994553amw    keyout[7] = keyin[6] & 0x7f;
3db3f65c6274eb042354801a308c8e9bc4994553amw
3db3f65c6274eb042354801a308c8e9bc4994553amw    for (i = 0; i < 8; i++)
3db3f65c6274eb042354801a308c8e9bc4994553amw        keyout[i] = (keyout[i] << 1) & 0xfe;
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw}
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross/*
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross * CKM_RC4
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rossint
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rosssmb_auth_RC4(uchar_t *Result, int ResultLen,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    uchar_t *Key, int KeyLen,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    uchar_t *Data, int DataLen)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross{
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    CK_RV rv;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    CK_MECHANISM mechanism;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    CK_OBJECT_HANDLE hKey;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    CK_SESSION_HANDLE hSession;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    CK_ULONG ciphertext_len;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    int error = SMBAUTH_FAILURE;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    /*
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * Use SUNW convenience function to initialize the cryptoki
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * library, and open a session with a slot that supports
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     * the mechanism we plan on using.
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross     */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    mechanism.mechanism = CKM_RC4;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    mechanism.pParameter = NULL;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    mechanism.ulParameterLen = 0;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (rv != CKR_OK) {
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        return (SMBAUTH_FAILURE);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    }
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        Key, KeyLen, &hKey);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (rv != CKR_OK)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        goto exit_session;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    /* Initialize the encryption operation in the session */
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    rv = C_EncryptInit(hSession, &mechanism, hKey);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (rv != CKR_OK)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        goto exit_encrypt;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    ciphertext_len = ResultLen;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    rv = C_EncryptUpdate(hSession,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        (CK_BYTE_PTR)Data, DataLen,
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        (CK_BYTE_PTR)Result, &ciphertext_len);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    if (rv == CKR_OK)
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross        error = 0;
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rossexit_encrypt:
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    (void) C_DestroyObject(hSession, hKey);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Rossexit_session:
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    (void) C_CloseSession(hSession);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross    return (error);
1ed6b69a5ca1ca3ee5e9a4931f74e2237c7e1c9fGordon Ross}