/* * Armv8-A Cryptographic Extension support functions for Aarch64 * * Copyright The Mbed TLS Contributors * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #if defined(__aarch64__) && !defined(__ARM_FEATURE_CRYPTO) && defined(__clang__) && __clang_major__ >= 4 /* TODO: Re-consider above after https://reviews.llvm.org/D131064 merged. * * The intrinsic declaration are guarded by predefined ACLE macros in clang: * these are normally only enabled by the -march option on the command line. * By defining the macros ourselves we gain access to those declarations without * requiring -march on the command line. * * `arm_neon.h` could be included by any header file, so we put these defines * at the top of this file, before any includes. */ #define __ARM_FEATURE_CRYPTO 1 /* See: https://arm-software.github.io/acle/main/acle.html#cryptographic-extensions * * `__ARM_FEATURE_CRYPTO` is deprecated, but we need to continue to specify it * for older compilers. */ #define __ARM_FEATURE_AES 1 #define MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG #endif #include #include "aesce.h" #if defined(MBEDTLS_AESCE_C) #if defined(MBEDTLS_HAVE_ARM64) /* Compiler version checks. */ #if defined(__clang__) # if __clang_major__ < 4 # error "Minimum version of Clang for MBEDTLS_AESCE_C is 4.0." # endif #elif defined(__GNUC__) # if __GNUC__ < 6 # error "Minimum version of GCC for MBEDTLS_AESCE_C is 6.0." # endif #elif defined(_MSC_VER) /* TODO: We haven't verified MSVC from 1920 to 1928. If someone verified that, * please update this and document of `MBEDTLS_AESCE_C` in * `mbedtls_config.h`. */ # if _MSC_VER < 1929 # error "Minimum version of MSVC for MBEDTLS_AESCE_C is 2019 version 16.11.2." # endif #endif #if !defined(__ARM_FEATURE_AES) || defined(MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG) # if defined(__clang__) # pragma clang attribute push (__attribute__((target("crypto"))), apply_to=function) # define MBEDTLS_POP_TARGET_PRAGMA # elif defined(__GNUC__) # pragma GCC push_options # pragma GCC target ("arch=armv8-a+crypto") # define MBEDTLS_POP_TARGET_PRAGMA # elif defined(_MSC_VER) # error "Required feature(__ARM_FEATURE_AES) is not enabled." # endif #endif /* !__ARM_FEATURE_AES || MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG */ #include #if defined(__linux__) #include #include #endif /* * AES instruction support detection routine */ int mbedtls_aesce_has_support(void) { #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); return (auxval & (HWCAP_ASIMD | HWCAP_AES)) == (HWCAP_ASIMD | HWCAP_AES); #else /* Assume AES instructions are supported. */ return 1; #endif } __attribute__((target("aes,crypto"))) static uint8x16_t aesce_encrypt_block(uint8x16_t block, unsigned char *keys, int rounds) { for (int i = 0; i < rounds - 1; i++) { /* AES AddRoundKey, SubBytes, ShiftRows (in this order). * AddRoundKey adds the round key for the previous round. */ block = vaeseq_u8(block, vld1q_u8(keys + i * 16)); /* AES mix columns */ block = vaesmcq_u8(block); } /* AES AddRoundKey for the previous round. * SubBytes, ShiftRows for the final round. */ block = vaeseq_u8(block, vld1q_u8(keys + (rounds -1) * 16)); /* Final round: no MixColumns */ /* Final AddRoundKey */ block = veorq_u8(block, vld1q_u8(keys + rounds * 16)); return block; } __attribute__((target("aes,crypto"))) static uint8x16_t aesce_decrypt_block(uint8x16_t block, unsigned char *keys, int rounds) { for (int i = 0; i < rounds - 1; i++) { /* AES AddRoundKey, SubBytes, ShiftRows */ block = vaesdq_u8(block, vld1q_u8(keys + i * 16)); /* AES inverse MixColumns for the next round. * * This means that we switch the order of the inverse AddRoundKey and * inverse MixColumns operations. We have to do this as AddRoundKey is * done in an atomic instruction together with the inverses of SubBytes * and ShiftRows. * * It works because MixColumns is a linear operation over GF(2^8) and * AddRoundKey is an exclusive or, which is equivalent to addition over * GF(2^8). (The inverse of MixColumns needs to be applied to the * affected round keys separately which has been done when the * decryption round keys were calculated.) */ block = vaesimcq_u8(block); } /* The inverses of AES AddRoundKey, SubBytes, ShiftRows finishing up the * last full round. */ block = vaesdq_u8(block, vld1q_u8(keys + (rounds - 1) * 16)); /* Inverse AddRoundKey for inverting the initial round key addition. */ block = veorq_u8(block, vld1q_u8(keys + rounds * 16)); return block; } /* * AES-ECB block en(de)cryption */ int mbedtls_aesce_crypt_ecb(mbedtls_aes_context *ctx, int mode, const unsigned char input[16], unsigned char output[16]) { uint8x16_t block = vld1q_u8(&input[0]); unsigned char *keys = (unsigned char *) (ctx->buf + ctx->rk_offset); if (mode == MBEDTLS_AES_ENCRYPT) { block = aesce_encrypt_block(block, keys, ctx->nr); } else { block = aesce_decrypt_block(block, keys, ctx->nr); } vst1q_u8(&output[0], block); return 0; } /* * Compute decryption round keys from encryption round keys */ __attribute__((target("aes,crypto"))) void mbedtls_aesce_inverse_key(unsigned char *invkey, const unsigned char *fwdkey, int nr) { int i, j; j = nr; vst1q_u8(invkey, vld1q_u8(fwdkey + j * 16)); for (i = 1, j--; j > 0; i++, j--) { vst1q_u8(invkey + i * 16, vaesimcq_u8(vld1q_u8(fwdkey + j * 16))); } vst1q_u8(invkey + i * 16, vld1q_u8(fwdkey + j * 16)); } static inline uint32_t aes_rot_word(uint32_t word) { return (word << (32 - 8)) | (word >> 8); } __attribute__((target("aes,crypto"))) static inline uint32_t aes_sub_word(uint32_t in) { uint8x16_t v = vreinterpretq_u8_u32(vdupq_n_u32(in)); uint8x16_t zero = vdupq_n_u8(0); /* vaeseq_u8 does both SubBytes and ShiftRows. Taking the first row yields * the correct result as ShiftRows doesn't change the first row. */ v = vaeseq_u8(zero, v); return vgetq_lane_u32(vreinterpretq_u32_u8(v), 0); } /* * Key expansion function */ static void aesce_setkey_enc(unsigned char *rk, const unsigned char *key, const size_t key_bit_length) { static uint8_t const rcon[] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; /* See https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.197.pdf * - Section 5, Nr = Nk + 6 * - Section 5.2, the length of round keys is Nb*(Nr+1) */ const uint32_t key_len_in_words = key_bit_length / 32; /* Nk */ const size_t round_key_len_in_words = 4; /* Nb */ const size_t rounds_needed = key_len_in_words + 6; /* Nr */ const size_t round_keys_len_in_words = round_key_len_in_words * (rounds_needed + 1); /* Nb*(Nr+1) */ const uint32_t *rko_end = (uint32_t *) rk + round_keys_len_in_words; memcpy(rk, key, key_len_in_words * 4); for (uint32_t *rki = (uint32_t *) rk; rki + key_len_in_words < rko_end; rki += key_len_in_words) { size_t iteration = (rki - (uint32_t *) rk) / key_len_in_words; uint32_t *rko; rko = rki + key_len_in_words; rko[0] = aes_rot_word(aes_sub_word(rki[key_len_in_words - 1])); rko[0] ^= rcon[iteration] ^ rki[0]; rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; if (rko + key_len_in_words > rko_end) { /* Do not write overflow words.*/ continue; } switch (key_bit_length) { case 128: break; case 192: rko[4] = rko[3] ^ rki[4]; rko[5] = rko[4] ^ rki[5]; break; case 256: rko[4] = aes_sub_word(rko[3]) ^ rki[4]; rko[5] = rko[4] ^ rki[5]; rko[6] = rko[5] ^ rki[6]; rko[7] = rko[6] ^ rki[7]; break; } } } /* * Key expansion, wrapper */ int mbedtls_aesce_setkey_enc(unsigned char *rk, const unsigned char *key, size_t bits) { switch (bits) { case 128: case 192: case 256: aesce_setkey_enc(rk, key, bits); break; default: return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; } return 0; } #if defined(MBEDTLS_POP_TARGET_PRAGMA) #if defined(__clang__) #pragma clang attribute pop #elif defined(__GNUC__) #pragma GCC pop_options #endif #undef MBEDTLS_POP_TARGET_PRAGMA #endif #endif /* MBEDTLS_HAVE_ARM64 */ #endif /* MBEDTLS_AESCE_C */