aes_intel.s revision 54034eb2d6e7d811adf4a1fe5105eac6fea6b0b5
/*
* ====================================================================
* Written by Intel Corporation for the OpenSSL project to add support
* for Intel AES-NI instructions. Rights for redistribution and usage
* in source and binary forms are granted according to the OpenSSL
* license.
*
* Author: Huang Ying <ying.huang at intel dot com>
* Vinodh Gopal <vinodh.gopal at intel dot com>
* Kahraman Akdemir
*
* Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD)
* instructions that are going to be introduced in the next generation
* of Intel processor, as of 2009. These instructions enable fast and
* secure data encryption and decryption, using the Advanced Encryption
* Standard (AES), defined by FIPS Publication number 197. The
* architecture introduces six instructions that offer full hardware
* support for AES. Four of them support high performance data
* encryption and decryption, and the other two instructions support
* the AES key expansion procedure.
* ====================================================================
*/
/*
* ====================================================================
* Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* distribution.
*
* 3. All advertising materials mentioning features or use of this
* software must display the following acknowleoudgment:
* "This product includes software developed by the OpenSSL Project
* for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
*
* 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For written permission, please contact
* openssl-core@openssl.org.
*
* 5. Products derived from this software may not be called "OpenSSL"
* nor may "OpenSSL" appear in their names without prior written
* permission of the OpenSSL Project.
*
* 6. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by the OpenSSL Project
* for use in the OpenSSL Toolkit (http://www.openssl.org/)"
*
* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
* EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
* ====================================================================
*/
/*
* ====================================================================
* OpenSolaris OS modifications
*
* This source originates as files aes-intel.S and eng_aesni_asm.pl, in
* patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
* Huang Ying of Intel to the openssl-dev mailing list under the subject
* of "Add support to Intel AES-NI instruction set for x86_64 platform".
*
* This OpenSolaris version has these major changes from the original source:
*
* /usr/include/sys/asm_linkage.h, lint(1B) guards, EXPORT DELETE START
* and EXPORT DELETE END markers, and dummy C function definitions for lint.
*
* 2. Formatted code, added comments, and added #includes and #defines.
*
* 3. Replaced aes* and palignr instructions with .byte sequences
* (as they are not supported yet by all of the gas, as, and aw assemblers).
*
* 4. If bit CR0.TS is set, clear and set the TS bit, after and before
* calling kpreempt_disable() and kpreempt_enable().
* If the TS bit is not set, Save and restore %xmm registers at the beginning
* and end of function calls (%xmm* registers are not saved and restored by
* during kernel thread preemption).
*
* 5. Renamed functions, reordered parameters, and changed return value
* to match OpenSolaris:
*
* OpenSSL interface:
* int intel_AES_set_encrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* int intel_AES_set_decrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return values for above are non-zero on error, 0 on success.
*
* void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
* void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
* typedef struct aes_key_st {
* unsigned int rd_key[4 *(AES_MAXNR + 1)];
* int rounds;
* unsigned int pad[3];
* } AES_KEY;
* Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules
* (ks32) instead of 64-bit (ks64).
* Number of rounds (aka round count) is at offset 240 of AES_KEY.
*
* OpenSolaris OS interface (#ifdefs removed for readability):
* int rijndael_key_setup_dec_intel(uint32_t rk[],
* const uint32_t cipherKey[], uint64_t keyBits);
* int rijndael_key_setup_enc_intel(uint32_t rk[],
* const uint32_t cipherKey[], uint64_t keyBits);
* Return values for above are 0 on error, number of rounds on success.
*
* void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4]);
* void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4]);
* typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4];
* uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t;
*
* typedef union {
* uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
* } aes_ks_t;
* typedef struct aes_key {
* aes_ks_t encr_ks, decr_ks;
* long double align128;
* int flags, nr, type;
* } aes_key_t;
*
* Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text,
* ct is crypto text, and MAX_AES_NR is 14.
* For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64.
* Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary.
* ====================================================================
*/
/* ARGSUSED */
void
}
/* ARGSUSED */
void
}
/* ARGSUSED */
int
return (0);
}
/* ARGSUSED */
int
return (0);
}
#else /* lint */
#include <sys/asm_linkage.h>
#include <sys/controlregs.h>
#ifdef _KERNEL
#include <sys/machprivregs.h>
#endif
#ifdef _KERNEL
/*
* Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is,
* it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
* uses it to pass P2 to syscall.
* This also occurs with the STTS macro, but we don't care if
* P2 (%rsi) is modified just before function exit.
* The CLTS and STTS macros push and pop P1 (%rdi) already.
*/
#ifdef __xpv
#define PROTECTED_CLTS \
CLTS; \
#else
#define PROTECTED_CLTS \
#endif /* __xpv */
#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) \
jnz 1f; \
jmp 2f; \
1: \
2:
/*
* If CR0_TS was not set above, pop %xmm0 and %xmm1 off stack,
* otherwise set CR0_TS.
*/
#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) \
jnz 1f; \
jmp 2f; \
1: \
2: \
/*
* If CR0_TS is not set, align stack (with push %rbp) and push
* %xmm0 - %xmm6 on stack, otherwise clear CR0_TS
*/
#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) \
jnz 1f; \
jmp 2f; \
1: \
2:
/*
* If CR0_TS was not set above, pop %xmm0 - %xmm6 off stack,
* otherwise set CR0_TS.
*/
#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) \
jnz 1f; \
jmp 2f; \
1: \
2: \
#else
#define PROTECTED_CLTS
#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg)
#define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
#endif /* _KERNEL */
/*
* _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
* _key_expansion_256a(), _key_expansion_256b()
*
* Helper functions called by rijndael_key_setup_inc_intel().
* Also used indirectly by rijndael_key_setup_dec_intel().
*
* Input:
* %xmm0 User-provided cipher key
* %xmm1 Round constant
* Output:
* (%rcx) AES key
*/
/* EXPORT DELETE START */
.align 16
.align 16
.align 16
.align 16
/* EXPORT DELETE END */
/*
* rijndael_key_setup_enc_intel()
* Expand the cipher key into the encryption key schedule.
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* OpenSolaris interface:
* int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
* uint64_t keyBits);
* Return value is 0 on error, number of rounds on success.
*
* Original Intel OpenSSL interface:
* int intel_AES_set_encrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return value is non-zero on error, 0 on success.
*/
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
#endif /* OPENSSL_INTERFACE */
/* EXPORT DELETE START */
#ifdef OPENSSL_INTERFACE
#endif /* OPENSSL_INTERFACE */
#ifdef OPENSSL_INTERFACE
#else /* Open Solaris Interface */
#endif
.align 4
#ifdef OPENSSL_INTERFACE
#endif /* OPENSSL_INTERFACE */
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
#endif
.align 4
#ifdef OPENSSL_INTERFACE
#endif /* OPENSSL_INTERFACE */
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
#endif
#ifdef OPENSSL_INTERFACE
#else
/* FALLTHROUGH */
#endif /* OPENSSL_INTERFACE */
#ifdef OPENSSL_INTERFACE
#else /* Open Solaris Interface */
#endif /* OPENSSL_INTERFACE */
/* EXPORT DELETE END */
/*
* rijndael_key_setup_dec_intel()
* Expand the cipher key into the decryption key schedule.
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* OpenSolaris interface:
* int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
* uint64_t keyBits);
* Return value is 0 on error, number of rounds on success.
* P1->P2, P2->P3, P3->P1
*
* Original Intel OpenSSL interface:
* int intel_AES_set_decrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return value is non-zero on error, 0 on success.
*/
/* EXPORT DELETE START */
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
#endif /* OPENSSL_INTERFACE */
#ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */
#endif
.align 4
.align 4
/* EXPORT DELETE END */
/*
* aes_encrypt_intel()
* Encrypt a single block (in and out can overlap).
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* Temporary register usage:
* %xmm0 State
* %xmm1 Key
*
* Original OpenSolaris Interface:
* void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4])
*
* Original Intel OpenSSL Interface:
* void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key)
*/
#ifdef OPENSSL_INTERFACE
#define aes_encrypt_intel intel_AES_encrypt
#define aes_decrypt_intel intel_AES_decrypt
/* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */
#else /* OpenSolaris Interface */
#endif /* OPENSSL_INTERFACE */
/* EXPORT DELETE START */
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
/* Round count is already present as P2 in %rsi/%esi */
#endif /* OPENSSL_INTERFACE */
/ AES 256
.align 4
.Lenc192:
.align 4
.Lenc128:
/* EXPORT DELETE END */
/*
* aes_decrypt_intel()
* Decrypt a single block (in and out can overlap).
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* Temporary register usage:
* %xmm0 State
* %xmm1 Key
*
* Original OpenSolaris Interface:
* void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4])/
*
* Original Intel OpenSSL Interface:
* void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
*/
/* EXPORT DELETE START */
#ifdef OPENSSL_INTERFACE
#else /* OpenSolaris Interface */
/* Round count is already present as P2 in %rsi/%esi */
#endif /* OPENSSL_INTERFACE */
/ AES 256
.align 4
.Ldec192:
.align 4
.Ldec128:
/* EXPORT DELETE END */
#endif /* lint || __lint */