RDP/client-1.8.3/ssl.c

a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* -*- c-basic-offset: 8 -*-
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   rdesktop: A Remote Desktop Protocol client.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   Secure sockets abstraction layer
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   Copyright (C) Matthew Chapman <matthewc.unsw.edu.au> 1999-2008
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   Copyright (C) Jay Sorg <j@american-data.com> 2006-2008
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   This program is free software: you can redistribute it and/or modify
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   it under the terms of the GNU General Public License as published by
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   the Free Software Foundation, either version 3 of the License, or
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   (at your option) any later version.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   This program is distributed in the hope that it will be useful,
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   but WITHOUT ANY WARRANTY; without even the implied warranty of
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   GNU General Public License for more details.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   You should have received a copy of the GNU General Public License
a180a41bba1d50822df23fff0099e90b86638b89vboxsync   along with this program.  If not, see <http://www.gnu.org/licenses/>.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync*/
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync/*
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * Oracle GPL Disclaimer: For the avoidance of doubt, except that if any license choice
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * other than GPL or LGPL is available it will apply instead, Oracle elects to use only
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * the General Public License version 2 (GPLv2) at this time for any software where
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * a choice of GPL license versions is made available with the language indicating
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * that GPLv2 or any later version may be used, or where a choice of which version
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync * of the GPL is applied is otherwise unspecified.
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync */
6e9aa255e3376b2da5824c09c4c62bc233463bfevboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync#include "rdesktop.h"
a180a41bba1d50822df23fff0099e90b86638b89vboxsync#include "ssl.h"
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_sha1_init(RDSSL_SHA1 * sha1)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    SHA1_Init(sha1);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_sha1_update(RDSSL_SHA1 * sha1, uint8 * data, uint32 len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    SHA1_Update(sha1, data, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_sha1_final(RDSSL_SHA1 * sha1, uint8 * out_data)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    SHA1_Final(out_data, sha1);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_md5_init(RDSSL_MD5 * md5)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    MD5_Init(md5);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_md5_update(RDSSL_MD5 * md5, uint8 * data, uint32 len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    MD5_Update(md5, data, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_md5_final(RDSSL_MD5 * md5, uint8 * out_data)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    MD5_Final(out_data, md5);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_rc4_set_key(RDSSL_RC4 * rc4, uint8 * key, uint32 len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    RC4_set_key(rc4, len, key);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_rc4_crypt(RDSSL_RC4 * rc4, uint8 * in_data, uint8 * out_data, uint32 len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    RC4(rc4, len, in_data, out_data);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncstatic void
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncreverse(uint8 * p, int len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    int i, j;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    uint8 temp;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    for (i = 0, j = len - 1; i < j; i++, j--)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    {
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        temp = p[i];
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        p[i] = p[j];
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        p[j] = temp;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    }
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * modulus,
a180a41bba1d50822df23fff0099e90b86638b89vboxsync          uint8 * exponent)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_CTX *ctx;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BIGNUM mod, exp, x, y;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    uint8 inr[SEC_MAX_MODULUS_SIZE];
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    int outlen;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(modulus, modulus_size);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(exponent, SEC_EXPONENT_SIZE);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    memcpy(inr, in, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(inr, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    ctx = BN_CTX_new();
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_init(&mod);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_init(&exp);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_init(&x);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_init(&y);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_bin2bn(modulus, modulus_size, &mod);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_bin2bn(inr, len, &x);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_mod_exp(&y, &x, &exp, &mod, ctx);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    outlen = BN_bn2bin(&y, out);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(out, outlen);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    if (outlen < (int) modulus_size)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        memset(out + outlen, 0, modulus_size - outlen);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_free(&y);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_clear_free(&x);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_free(&exp);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_free(&mod);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    BN_CTX_free(ctx);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* returns newly allocated RDSSL_CERT or NULL */
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncRDSSL_CERT *
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_cert_read(uint8 * data, uint32 len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    /* this will move the data pointer but we don't care, we don't use it again */
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return d2i_X509(NULL, (D2I_X509_CONST unsigned char **) &data, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_cert_free(RDSSL_CERT * cert)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    X509_free(cert);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* returns newly allocated RDSSL_RKEY or NULL */
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncRDSSL_RKEY *
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    EVP_PKEY *epk = NULL;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    RDSSL_RKEY *lkey;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    int nid;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    /* By some reason, Microsoft sets the OID of the Public RSA key to
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       Kudos to Richard Levitte for the following (. intiutive .)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       lines of code that resets the OID and let's us extract the key. */
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    {
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    }
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    epk = X509_get_pubkey(cert);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    if (NULL == epk)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    {
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        error("Failed to extract public key from certificate\n");
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        return NULL;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    }
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    lkey = RSAPublicKey_dup(EVP_PKEY_get1_RSA(epk));
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    EVP_PKEY_free(epk);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    *key_len = RSA_size(lkey);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return lkey;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* returns boolean */
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncRD_BOOL
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_certs_ok(RDSSL_CERT * server_cert, RDSSL_CERT * cacert)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    /* Currently, we don't use the CA Certificate.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       FIXME:
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       *) Verify the server certificate (server_cert) with the
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       CA certificate.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       *) Store the CA Certificate with the hostname of the
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       server we are connecting to as key, and compare it
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       when we connect the next time, in order to prevent
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       MITM-attacks.
a180a41bba1d50822df23fff0099e90b86638b89vboxsync     */
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return True;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncint
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_cert_print_fp(FILE * fp, RDSSL_CERT * cert)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return X509_print_fp(fp, cert);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_rkey_free(RDSSL_RKEY * rkey)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    RSA_free(rkey);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* returns error */
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncint
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
a180a41bba1d50822df23fff0099e90b86638b89vboxsync               uint32 max_mod_len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    int len;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        (BN_num_bytes(rkey->n) > (int) max_mod_len))
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    {
a180a41bba1d50822df23fff0099e90b86638b89vboxsync        return 1;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    }
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    len = BN_bn2bin(rkey->e, exponent);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(exponent, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    len = BN_bn2bin(rkey->n, modulus);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    reverse(modulus, len);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return 0;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync/* returns boolean */
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncRD_BOOL
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_sig_ok(uint8 * exponent, uint32 exp_len, uint8 * modulus, uint32 mod_len,
a180a41bba1d50822df23fff0099e90b86638b89vboxsync         uint8 * signature, uint32 sig_len)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    /* Currently, we don't check the signature
a180a41bba1d50822df23fff0099e90b86638b89vboxsync       FIXME:
a180a41bba1d50822df23fff0099e90b86638b89vboxsync     */
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    return True;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsync
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncvoid
a180a41bba1d50822df23fff0099e90b86638b89vboxsyncrdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
a180a41bba1d50822df23fff0099e90b86638b89vboxsync           unsigned char *md)
a180a41bba1d50822df23fff0099e90b86638b89vboxsync{
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    HMAC_CTX ctx;
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    HMAC_CTX_init(&ctx);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync    HMAC_CTX_cleanup(&ctx);
a180a41bba1d50822df23fff0099e90b86638b89vboxsync}