/*
* ====================================================================
* Written by Intel Corporation for the OpenSSL project to add support
* for Intel AES-NI instructions. Rights for redistribution and usage
* in source and binary forms are granted according to the OpenSSL
* license.
*
* Author: Huang Ying <ying.huang at intel dot com>
* Vinodh Gopal <vinodh.gopal at intel dot com>
* Kahraman Akdemir
*
* Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD)
* instructions that are going to be introduced in the next generation
* of Intel processor, as of 2009. These instructions enable fast and
* secure data encryption and decryption, using the Advanced Encryption
* Standard (AES), defined by FIPS Publication number 197. The
* architecture introduces six instructions that offer full hardware
* support for AES. Four of them support high performance data
* encryption and decryption, and the other two instructions support
* the AES key expansion procedure.
* ====================================================================
*/
/*
* ====================================================================
* Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. All advertising materials mentioning features or use of this
* software must display the following acknowledgment:
* "This product includes software developed by the OpenSSL Project
* for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
*
* 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For written permission, please contact
* openssl-core@openssl.org.
*
* 5. Products derived from this software may not be called "OpenSSL"
* nor may "OpenSSL" appear in their names without prior written
* permission of the OpenSSL Project.
*
* 6. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by the OpenSSL Project
* for use in the OpenSSL Toolkit (http://www.openssl.org/)"
*
* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
* EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
* ====================================================================
*/
/*
* ====================================================================
* OpenSolaris OS modifications
*
* This source originates as files aes-intel.S and eng_aesni_asm.pl, in
* patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
* Huang Ying of Intel to the openssl-dev mailing list under the subject
* of "Add support to Intel AES-NI instruction set for x86_64 platform".
*
* This OpenSolaris version has these major changes from the original source:
*
* 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
* /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
* definitions for lint.
*
* 2. Formatted code, added comments, and added #includes and #defines.
*
* 3. If bit CR0.TS is set, clear and set the TS bit, after and before
* calling kpreempt_disable() and kpreempt_enable().
* If the TS bit is not set, Save and restore %xmm registers at the beginning
* and end of function calls (%xmm* registers are not saved and restored by
* during kernel thread preemption).
*
* 4. Renamed functions, reordered parameters, and changed return value
* to match OpenSolaris:
*
* OpenSSL interface:
* int intel_AES_set_encrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* int intel_AES_set_decrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return values for above are non-zero on error, 0 on success.
*
* void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
* void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
* typedef struct aes_key_st {
* unsigned int rd_key[4 *(AES_MAXNR + 1)];
* int rounds;
* unsigned int pad[3];
* } AES_KEY;
* Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules
* (ks32) instead of 64-bit (ks64).
* Number of rounds (aka round count) is at offset 240 of AES_KEY.
*
* OpenSolaris OS interface (#ifdefs removed for readability):
* int rijndael_key_setup_dec_intel(uint32_t rk[],
* const uint32_t cipherKey[], uint64_t keyBits);
* int rijndael_key_setup_enc_intel(uint32_t rk[],
* const uint32_t cipherKey[], uint64_t keyBits);
* Return values for above are 0 on error, number of rounds on success.
*
* void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4]);
* void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4]);
* typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4];
* uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t;
*
* typedef union {
* uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
* } aes_ks_t;
* typedef struct aes_key {
* aes_ks_t encr_ks, decr_ks;
* long double align128;
* int flags, nr, type;
* } aes_key_t;
*
* Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text,
* ct is crypto text, and MAX_AES_NR is 14.
* For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64.
*
* Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary.
*
* ====================================================================
*/
#if defined(lint) || defined(__lint)
#include <sys/types.h>
/* ARGSUSED */
void
aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4],
uint32_t ct[4]) {
}
/* ARGSUSED */
void
aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4],
uint32_t pt[4]) {
}
/* ARGSUSED */
int
rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
uint64_t keyBits) {
return (0);
}
/* ARGSUSED */
int
rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
uint64_t keyBits) {
return (0);
}
#else /* lint */
#include <sys/asm_linkage.h>
#include <sys/controlregs.h>
#ifdef _KERNEL
#include <sys/machprivregs.h>
#endif
#ifdef _KERNEL
/*
* Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is,
* it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
* uses it to pass P2 to syscall.
* This also occurs with the STTS macro, but we don't care if
* P2 (%rsi) is modified just before function exit.
* The CLTS and STTS macros push and pop P1 (%rdi) already.
*/
#ifdef __xpv
#define PROTECTED_CLTS \
push %rsi; \
CLTS; \
pop %rsi
#else
#define PROTECTED_CLTS \
CLTS
#endif /* __xpv */
#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) \
push %rbp; \
mov %rsp, %rbp; \
movq %cr0, tmpreg; \
testq $CR0_TS, tmpreg; \
jnz 1f; \
and $-XMM_ALIGN, %rsp; \
sub $[XMM_SIZE * 2], %rsp; \
movaps %xmm0, 16(%rsp); \
movaps %xmm1, (%rsp); \
jmp 2f; \
1: \
PROTECTED_CLTS; \
2:
/*
* If CR0_TS was not set above, pop %xmm0 and %xmm1 off stack,
* otherwise set CR0_TS.
*/
#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) \
testq $CR0_TS, tmpreg; \
jnz 1f; \
movaps (%rsp), %xmm1; \
movaps 16(%rsp), %xmm0; \
jmp 2f; \
1: \
STTS(tmpreg); \
2: \
mov %rbp, %rsp; \
pop %rbp
/*
* If CR0_TS is not set, align stack (with push %rbp) and push
* %xmm0 - %xmm6 on stack, otherwise clear CR0_TS
*/
#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) \
push %rbp; \
mov %rsp, %rbp; \
movq %cr0, tmpreg; \
testq $CR0_TS, tmpreg; \
jnz 1f; \
and $-XMM_ALIGN, %rsp; \
sub $[XMM_SIZE * 7], %rsp; \
movaps %xmm0, 96(%rsp); \
movaps %xmm1, 80(%rsp); \
movaps %xmm2, 64(%rsp); \
movaps %xmm3, 48(%rsp); \
movaps %xmm4, 32(%rsp); \
movaps %xmm5, 16(%rsp); \
movaps %xmm6, (%rsp); \
jmp 2f; \
1: \
PROTECTED_CLTS; \
2:
/*
* If CR0_TS was not set above, pop %xmm0 - %xmm6 off stack,
* otherwise set CR0_TS.
*/
#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) \
testq $CR0_TS, tmpreg; \
jnz 1f; \
movaps (%rsp), %xmm6; \
movaps 16(%rsp), %xmm5; \
movaps 32(%rsp), %xmm4; \
movaps 48(%rsp), %xmm3; \
movaps 64(%rsp), %xmm2; \
movaps 80(%rsp), %xmm1; \
movaps 96(%rsp), %xmm0; \
jmp 2f; \
1: \
STTS(tmpreg); \
2: \
mov %rbp, %rsp; \
pop %rbp
#else
#define PROTECTED_CLTS
#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg)
#define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
#endif /* _KERNEL */
/*
* _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
* _key_expansion_256a(), _key_expansion_256b()
*
* Helper functions called by rijndael_key_setup_inc_intel().
* Also used indirectly by rijndael_key_setup_dec_intel().
*
* Input:
* %xmm0 User-provided cipher key
* %xmm1 Round constant
* Output:
* (%rcx) AES key
*/
.align 16
_key_expansion_128:
_key_expansion_256a:
pshufd $0b11111111, %xmm1, %xmm1
shufps $0b00010000, %xmm0, %xmm4
pxor %xmm4, %xmm0
shufps $0b10001100, %xmm0, %xmm4
pxor %xmm4, %xmm0
pxor %xmm1, %xmm0
movaps %xmm0, (%rcx)
add $0x10, %rcx
ret
SET_SIZE(_key_expansion_128)
SET_SIZE(_key_expansion_256a)
.align 16
_key_expansion_192a:
pshufd $0b01010101, %xmm1, %xmm1
shufps $0b00010000, %xmm0, %xmm4
pxor %xmm4, %xmm0
shufps $0b10001100, %xmm0, %xmm4
pxor %xmm4, %xmm0
pxor %xmm1, %xmm0
movaps %xmm2, %xmm5
movaps %xmm2, %xmm6
pslldq $4, %xmm5
pshufd $0b11111111, %xmm0, %xmm3
pxor %xmm3, %xmm2
pxor %xmm5, %xmm2
movaps %xmm0, %xmm1
shufps $0b01000100, %xmm0, %xmm6
movaps %xmm6, (%rcx)
shufps $0b01001110, %xmm2, %xmm1
movaps %xmm1, 0x10(%rcx)
add $0x20, %rcx
ret
SET_SIZE(_key_expansion_192a)
.align 16
_key_expansion_192b:
pshufd $0b01010101, %xmm1, %xmm1
shufps $0b00010000, %xmm0, %xmm4
pxor %xmm4, %xmm0
shufps $0b10001100, %xmm0, %xmm4
pxor %xmm4, %xmm0
pxor %xmm1, %xmm0
movaps %xmm2, %xmm5
pslldq $4, %xmm5
pshufd $0b11111111, %xmm0, %xmm3
pxor %xmm3, %xmm2
pxor %xmm5, %xmm2
movaps %xmm0, (%rcx)
add $0x10, %rcx
ret
SET_SIZE(_key_expansion_192b)
.align 16
_key_expansion_256b:
pshufd $0b10101010, %xmm1, %xmm1
shufps $0b00010000, %xmm2, %xmm4
pxor %xmm4, %xmm2
shufps $0b10001100, %xmm2, %xmm4
pxor %xmm4, %xmm2
pxor %xmm1, %xmm2
movaps %xmm2, (%rcx)
add $0x10, %rcx
ret
SET_SIZE(_key_expansion_256b)
/*
* rijndael_key_setup_enc_intel()
* Expand the cipher key into the encryption key schedule.
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* has been called. This is because %xmm registers are not saved/restored.
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* OpenSolaris interface:
* int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
* uint64_t keyBits);
* Return value is 0 on error, number of rounds on success.
*
* Original Intel OpenSSL interface:
* int intel_AES_set_encrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return value is non-zero on error, 0 on success.
*/
#ifdef OPENSSL_INTERFACE
#define rijndael_key_setup_enc_intel intel_AES_set_encrypt_key
#define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key
#define USERCIPHERKEY rdi /* P1, 64 bits */
#define KEYSIZE32 esi /* P2, 32 bits */
#define KEYSIZE64 rsi /* P2, 64 bits */
#define AESKEY rdx /* P3, 64 bits */
#else /* OpenSolaris Interface */
#define AESKEY rdi /* P1, 64 bits */
#define USERCIPHERKEY rsi /* P2, 64 bits */
#define KEYSIZE32 edx /* P3, 32 bits */
#define KEYSIZE64 rdx /* P3, 64 bits */
#endif /* OPENSSL_INTERFACE */
#define ROUNDS32 KEYSIZE32 /* temp */
#define ROUNDS64 KEYSIZE64 /* temp */
#define ENDAESKEY USERCIPHERKEY /* temp */
ENTRY_NP(rijndael_key_setup_enc_intel)
CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
/ NULL pointer sanity check
test %USERCIPHERKEY, %USERCIPHERKEY
jz .Lenc_key_invalid_param
test %AESKEY, %AESKEY
jz .Lenc_key_invalid_param
movups (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
movaps %xmm0, (%AESKEY)
lea 0x10(%AESKEY), %rcx / key addr
pxor %xmm4, %xmm4 / xmm4 is assumed 0 in _key_expansion_x
cmp $256, %KEYSIZE32
jnz .Lenc_key192
/ AES 256: 14 rounds in encryption key schedule
#ifdef OPENSSL_INTERFACE
mov $14, %ROUNDS32
movl %ROUNDS32, 240(%AESKEY) / key.rounds = 14
#endif /* OPENSSL_INTERFACE */
movups 0x10(%USERCIPHERKEY), %xmm2 / other user key (2nd 16 bytes)
movaps %xmm2, (%rcx)
add $0x10, %rcx
aeskeygenassist $0x1, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x1, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x2, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x2, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x4, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x4, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x8, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x8, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x10, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x10, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x20, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
aeskeygenassist $0x20, %xmm0, %xmm1
call _key_expansion_256b
aeskeygenassist $0x40, %xmm2, %xmm1 / expand the key
call _key_expansion_256a
SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
#ifdef OPENSSL_INTERFACE
xor %rax, %rax / return 0 (OK)
#else /* Open Solaris Interface */
mov $14, %rax / return # rounds = 14
#endif
ret
.align 4
.Lenc_key192:
cmp $192, %KEYSIZE32
jnz .Lenc_key128
/ AES 192: 12 rounds in encryption key schedule
#ifdef OPENSSL_INTERFACE
mov $12, %ROUNDS32
movl %ROUNDS32, 240(%AESKEY) / key.rounds = 12
#endif /* OPENSSL_INTERFACE */
movq 0x10(%USERCIPHERKEY), %xmm2 / other user key
aeskeygenassist $0x1, %xmm2, %xmm1 / expand the key
call _key_expansion_192a
aeskeygenassist $0x2, %xmm2, %xmm1 / expand the key
call _key_expansion_192b
aeskeygenassist $0x4, %xmm2, %xmm1 / expand the key
call _key_expansion_192a
aeskeygenassist $0x8, %xmm2, %xmm1 / expand the key
call _key_expansion_192b
aeskeygenassist $0x10, %xmm2, %xmm1 / expand the key
call _key_expansion_192a
aeskeygenassist $0x20, %xmm2, %xmm1 / expand the key
call _key_expansion_192b
aeskeygenassist $0x40, %xmm2, %xmm1 / expand the key
call _key_expansion_192a
aeskeygenassist $0x80, %xmm2, %xmm1 / expand the key
call _key_expansion_192b
SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
#ifdef OPENSSL_INTERFACE
xor %rax, %rax / return 0 (OK)
#else /* OpenSolaris Interface */
mov $12, %rax / return # rounds = 12
#endif
ret
.align 4
.Lenc_key128:
cmp $128, %KEYSIZE32
jnz .Lenc_key_invalid_key_bits
/ AES 128: 10 rounds in encryption key schedule
#ifdef OPENSSL_INTERFACE
mov $10, %ROUNDS32
movl %ROUNDS32, 240(%AESKEY) / key.rounds = 10
#endif /* OPENSSL_INTERFACE */
aeskeygenassist $0x1, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x2, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x4, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x8, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x10, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x20, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x40, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x80, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x1b, %xmm0, %xmm1 / expand the key
call _key_expansion_128
aeskeygenassist $0x36, %xmm0, %xmm1 / expand the key
call _key_expansion_128
SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
#ifdef OPENSSL_INTERFACE
xor %rax, %rax / return 0 (OK)
#else /* OpenSolaris Interface */
mov $10, %rax / return # rounds = 10
#endif
ret
.Lenc_key_invalid_param:
#ifdef OPENSSL_INTERFACE
SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
mov $-1, %rax / user key or AES key pointer is NULL
ret
#else
/* FALLTHROUGH */
#endif /* OPENSSL_INTERFACE */
.Lenc_key_invalid_key_bits:
SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
#ifdef OPENSSL_INTERFACE
mov $-2, %rax / keysize is invalid
#else /* Open Solaris Interface */
xor %rax, %rax / a key pointer is NULL or invalid keysize
#endif /* OPENSSL_INTERFACE */
ret
SET_SIZE(rijndael_key_setup_enc_intel)
/*
* rijndael_key_setup_dec_intel()
* Expand the cipher key into the decryption key schedule.
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* has been called. This is because %xmm registers are not saved/restored.
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* OpenSolaris interface:
* int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
* uint64_t keyBits);
* Return value is 0 on error, number of rounds on success.
* P1->P2, P2->P3, P3->P1
*
* Original Intel OpenSSL interface:
* int intel_AES_set_decrypt_key(const unsigned char *userKey,
* const int bits, AES_KEY *key);
* Return value is non-zero on error, 0 on success.
*/
ENTRY_NP(rijndael_key_setup_dec_intel)
/ Generate round keys used for encryption
call rijndael_key_setup_enc_intel
test %rax, %rax
#ifdef OPENSSL_INTERFACE
jnz .Ldec_key_exit / Failed if returned non-0
#else /* OpenSolaris Interface */
jz .Ldec_key_exit / Failed if returned 0
#endif /* OPENSSL_INTERFACE */
CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
/*
* Convert round keys used for encryption
* to a form usable for decryption
*/
#ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */
mov %rax, %ROUNDS64 / set # rounds (10, 12, or 14)
/ (already set for OpenSSL)
#endif
lea 0x10(%AESKEY), %rcx / key addr
shl $4, %ROUNDS32
add %AESKEY, %ROUNDS64
mov %ROUNDS64, %ENDAESKEY
.align 4
.Ldec_key_reorder_loop:
movaps (%AESKEY), %xmm0
movaps (%ROUNDS64), %xmm1
movaps %xmm0, (%ROUNDS64)
movaps %xmm1, (%AESKEY)
lea 0x10(%AESKEY), %AESKEY
lea -0x10(%ROUNDS64), %ROUNDS64
cmp %AESKEY, %ROUNDS64
ja .Ldec_key_reorder_loop
.align 4
.Ldec_key_inv_loop:
movaps (%rcx), %xmm0
/ Convert an encryption round key to a form usable for decryption
/ with the "AES Inverse Mix Columns" instruction
aesimc %xmm0, %xmm1
movaps %xmm1, (%rcx)
lea 0x10(%rcx), %rcx
cmp %ENDAESKEY, %rcx
jnz .Ldec_key_inv_loop
SET_TS_OR_POP_XMM0_XMM1(%r10)
.Ldec_key_exit:
/ OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
/ OpenSSL: rax = 0 for OK, or non-zero for error
ret
SET_SIZE(rijndael_key_setup_dec_intel)
/*
* aes_encrypt_intel()
* Encrypt a single block (in and out can overlap).
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* has been called. This is because %xmm registers are not saved/restored.
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* Temporary register usage:
* %xmm0 State
* %xmm1 Key
*
* Original OpenSolaris Interface:
* void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4])
*
* Original Intel OpenSSL Interface:
* void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key)
*/
#ifdef OPENSSL_INTERFACE
#define aes_encrypt_intel intel_AES_encrypt
#define aes_decrypt_intel intel_AES_decrypt
#define INP rdi /* P1, 64 bits */
#define OUTP rsi /* P2, 64 bits */
#define KEYP rdx /* P3, 64 bits */
/* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */
#define NROUNDS32 ecx /* temporary, 32 bits */
#define NROUNDS cl /* temporary, 8 bits */
#else /* OpenSolaris Interface */
#define KEYP rdi /* P1, 64 bits */
#define NROUNDS esi /* P2, 32 bits */
#define INP rdx /* P3, 64 bits */
#define OUTP rcx /* P4, 64 bits */
#endif /* OPENSSL_INTERFACE */
#define STATE xmm0 /* temporary, 128 bits */
#define KEY xmm1 /* temporary, 128 bits */
ENTRY_NP(aes_encrypt_intel)
CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
movups (%INP), %STATE / input
movaps (%KEYP), %KEY / key
#ifdef OPENSSL_INTERFACE
mov 240(%KEYP), %NROUNDS32 / round count
#else /* OpenSolaris Interface */
/* Round count is already present as P2 in %rsi/%esi */
#endif /* OPENSSL_INTERFACE */
pxor %KEY, %STATE / round 0
lea 0x30(%KEYP), %KEYP
cmp $12, %NROUNDS
jb .Lenc128
lea 0x20(%KEYP), %KEYP
je .Lenc192
/ AES 256
lea 0x20(%KEYP), %KEYP
movaps -0x60(%KEYP), %KEY
aesenc %KEY, %STATE
movaps -0x50(%KEYP), %KEY
aesenc %KEY, %STATE
.align 4
.Lenc192:
/ AES 192 and 256
movaps -0x40(%KEYP), %KEY
aesenc %KEY, %STATE
movaps -0x30(%KEYP), %KEY
aesenc %KEY, %STATE
.align 4
.Lenc128:
/ AES 128, 192, and 256
movaps -0x20(%KEYP), %KEY
aesenc %KEY, %STATE
movaps -0x10(%KEYP), %KEY
aesenc %KEY, %STATE
movaps (%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x10(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x20(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x30(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x40(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x50(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x60(%KEYP), %KEY
aesenc %KEY, %STATE
movaps 0x70(%KEYP), %KEY
aesenclast %KEY, %STATE / last round
movups %STATE, (%OUTP) / output
SET_TS_OR_POP_XMM0_XMM1(%r10)
ret
SET_SIZE(aes_encrypt_intel)
/*
* aes_decrypt_intel()
* Decrypt a single block (in and out can overlap).
*
* For kernel code, caller is responsible for ensuring kpreempt_disable()
* has been called. This is because %xmm registers are not saved/restored.
* Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
* on entry. Otherwise, if TS is not set, save and restore %xmm registers
* on the stack.
*
* Temporary register usage:
* %xmm0 State
* %xmm1 Key
*
* Original OpenSolaris Interface:
* void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
* const uint32_t pt[4], uint32_t ct[4])/
*
* Original Intel OpenSSL Interface:
* void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
* const AES_KEY *key);
*/
ENTRY_NP(aes_decrypt_intel)
CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
movups (%INP), %STATE / input
movaps (%KEYP), %KEY / key
#ifdef OPENSSL_INTERFACE
mov 240(%KEYP), %NROUNDS32 / round count
#else /* OpenSolaris Interface */
/* Round count is already present as P2 in %rsi/%esi */
#endif /* OPENSSL_INTERFACE */
pxor %KEY, %STATE / round 0
lea 0x30(%KEYP), %KEYP
cmp $12, %NROUNDS
jb .Ldec128
lea 0x20(%KEYP), %KEYP
je .Ldec192
/ AES 256
lea 0x20(%KEYP), %KEYP
movaps -0x60(%KEYP), %KEY
aesdec %KEY, %STATE
movaps -0x50(%KEYP), %KEY
aesdec %KEY, %STATE
.align 4
.Ldec192:
/ AES 192 and 256
movaps -0x40(%KEYP), %KEY
aesdec %KEY, %STATE
movaps -0x30(%KEYP), %KEY
aesdec %KEY, %STATE
.align 4
.Ldec128:
/ AES 128, 192, and 256
movaps -0x20(%KEYP), %KEY
aesdec %KEY, %STATE
movaps -0x10(%KEYP), %KEY
aesdec %KEY, %STATE
movaps (%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x10(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x20(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x30(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x40(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x50(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x60(%KEYP), %KEY
aesdec %KEY, %STATE
movaps 0x70(%KEYP), %KEY
aesdeclast %KEY, %STATE / last round
movups %STATE, (%OUTP) / output
SET_TS_OR_POP_XMM0_XMM1(%r10)
ret
SET_SIZE(aes_decrypt_intel)
#endif /* lint || __lint */