common/crypto/pkix-verify.cpp

13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync/* $Id$ */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync/** @file
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * IPRT - Crypto - Public Key Infrastructure API, Verification.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync/*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * Copyright (C) 2006-2014 Oracle Corporation
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync *
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * This file is part of VirtualBox Open Source Edition (OSE), as
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * available from http://www.virtualbox.org. This file is free software;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * you can redistribute it and/or modify it under the terms of the GNU
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * General Public License (GPL) as published by the Free Software
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * Foundation, in version 2 as it comes in the "COPYING" file of the
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync *
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * The contents of this file may alternatively be used under the terms
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * of the Common Development and Distribution License Version 1.0
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * VirtualBox OSE distribution, in which case the provisions of the
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * CDDL are applicable instead of those of the GPL.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync *
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * You may elect to license modified versions of this file under the
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync * terms and conditions of either the GPL or the CDDL or both.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync/*******************************************************************************
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync*   Header Files                                                               *
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync*******************************************************************************/
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#include "internal/iprt.h"
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#include <iprt/crypto/pkix.h>
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#include <iprt/err.h>
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#include <iprt/string.h>
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#include <iprt/crypto/digest.h>
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#ifdef IPRT_WITH_OPENSSL
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync# include "internal/iprt-openssl.h"
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync# include "openssl/evp.h"
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#endif
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsyncRTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNTYPE pParameters, PCRTASN1BITSTRING pPublicKey,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                          PCRTASN1BITSTRING pSignatureValue, const void *pvData, size_t cbData,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                          PRTERRINFO pErrInfo)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync{
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     * Valid input.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertPtrReturn(pAlgorithm, VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertReturn(RTAsn1ObjId_IsPresent(pAlgorithm), VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (pParameters)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        AssertPtrReturn(pParameters, VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        if (pParameters->enmType == RTASN1TYPE_NULL)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            pParameters = NULL;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    }
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertPtrReturn(pPublicKey, VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertReturn(RTAsn1BitString_IsPresent(pPublicKey), VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertPtrReturn(pSignatureValue, VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertReturn(RTAsn1BitString_IsPresent(pSignatureValue), VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertPtrReturn(pvData, VERR_INVALID_POINTER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertReturn(cbData > 0, VERR_INVALID_PARAMETER);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     * Parameters are not currently supported (openssl code path).
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (pParameters)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return RTErrInfoSet(pErrInfo, VERR_CR_PKIX_CIPHER_ALGO_PARAMS_NOT_IMPL,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                            "Cipher algorithm parameters are not yet supported.");
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     * Validate using IPRT.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    RTCRPKIXSIGNATURE hSignature;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    int rcIprt = RTCrPkixSignatureCreateByObjId(&hSignature, pAlgorithm, false /*fSigning*/, pPublicKey, pParameters);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (RT_FAILURE(rcIprt))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_CIPHER_ALGO_NOT_KNOWN,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                             "Unknown public key algorithm [IPRT]: %s", pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    RTCRDIGEST hDigest;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    rcIprt = RTCrDigestCreateByObjId(&hDigest, pAlgorithm);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (RT_SUCCESS(rcIprt))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        /* Calculate the digest. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        rcIprt = RTCrDigestUpdate(hDigest, pvData, cbData);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        if (RT_SUCCESS(rcIprt))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            rcIprt = RTCrPkixSignatureVerifyBitString(hSignature, hDigest, pSignatureValue);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            if (RT_FAILURE(rcIprt))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                RTErrInfoSet(pErrInfo, rcIprt, "RTCrPkixSignatureVerifyBitString failed");
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        }
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        else
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            RTErrInfoSet(pErrInfo, rcIprt, "RTCrDigestUpdate failed");
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        RTCrDigestRelease(hDigest);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    }
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    else
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        RTErrInfoSetF(pErrInfo, rcIprt, "Unknown digest algorithm [IPRT]: %s", pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    RTCrPkixSignatureRelease(hSignature);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#ifdef IPRT_WITH_OPENSSL
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     * Validate using OpenSSL EVP.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    rtCrOpenSslInit();
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /* Translate the algorithm ID into a EVP message digest type pointer. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    int iAlgoNid = OBJ_txt2nid(pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (iAlgoNid == NID_undef)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                             "Unknown public key algorithm [OpenSSL]: %s", pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    const char *pszAlogSn = OBJ_nid2sn(iAlgoNid);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    const EVP_MD *pEvpMdType = EVP_get_digestbyname(pszAlogSn);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (!pEvpMdType)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN_EVP,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                             "EVP_get_digestbyname failed on %s (%s)", pszAlogSn, pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /* Initialize the EVP message digest context. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    EVP_MD_CTX EvpMdCtx;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    EVP_MD_CTX_init(&EvpMdCtx);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (!EVP_VerifyInit_ex(&EvpMdCtx, pEvpMdType, NULL /*engine*/))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                             "EVP_VerifyInit_ex failed (algorithm type is %s / %s)", pszAlogSn, pAlgorithm->szObjId);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /* Create an EVP public key. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    int rcOssl;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (pEvpPublicKey)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        if (pEvpPublicKey->type != NID_undef)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            {
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                /* Digest the data. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                EVP_VerifyUpdate(&EvpMdCtx, pvData, cbData);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                /* Verify the signature. */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                if (EVP_VerifyFinal(&EvpMdCtx,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                    RTASN1BITSTRING_GET_BIT0_PTR(pSignatureValue),
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                    RTASN1BITSTRING_GET_BYTE_SIZE(pSignatureValue),
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                    pEvpPublicKey) > 0)
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                    rcOssl = VINF_SUCCESS;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                else
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                    rcOssl = RTErrInfoSet(pErrInfo, VERR_CR_PKIX_OSSL_VERIFY_FINAL_FAILED, "EVP_VerifyFinal failed");
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            }
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            else
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                rcOssl = RTErrInfoSet(pErrInfo, VERR_CR_PKIX_OSSL_D2I_PUBLIC_KEY_FAILED, "d2i_PublicKey failed");
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        }
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        else
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync            rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync                                   "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        /* Cleanup and return.*/
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        EVP_PKEY_free(pEvpPublicKey);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    }
e5996b7752d36e94746f991da154f641056528d8vboxsync    else
e5996b7752d36e94746f991da154f641056528d8vboxsync        rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    EVP_MD_CTX_cleanup(&EvpMdCtx);
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    /*
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     * Check the result.
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync     */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (RT_SUCCESS(rcIprt) && RT_SUCCESS(rcOssl))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return VINF_SUCCESS;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (RT_FAILURE_NP(rcIprt) && RT_FAILURE_NP(rcOssl))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return rcIprt;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    AssertMsgFailed(("rcIprt=%Rrc rcOssl=%Rrc\n", rcIprt, rcOssl));
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    if (RT_FAILURE_NP(rcOssl))
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync        return rcOssl;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync#endif /* IPRT_WITH_OPENSSL */
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync    return rcIprt;
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync}
13493ab7596e827b8d0caab2c89e635dd65f78f9vboxsync