diff options
Diffstat (limited to 'security/nss/lib/freebl/gcm.c')
| -rw-r--r-- | security/nss/lib/freebl/gcm.c | 171 |
1 files changed, 122 insertions, 49 deletions
diff --git a/security/nss/lib/freebl/gcm.c b/security/nss/lib/freebl/gcm.c index f1e16da78..0fdb0fd48 100644 --- a/security/nss/lib/freebl/gcm.c +++ b/security/nss/lib/freebl/gcm.c @@ -17,50 +17,18 @@ #include <limits.h> +#ifdef NSS_X86_OR_X64 +#include <wmmintrin.h> /* clmul */ +#endif + /* Forward declarations */ -SECStatus gcm_HashInit_hw(gcmHashContext *ghash); -SECStatus gcm_HashWrite_hw(gcmHashContext *ghash, unsigned char *outbuf); SECStatus gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf, unsigned int count); -SECStatus gcm_HashZeroX_hw(gcmHashContext *ghash); SECStatus gcm_HashMult_sftw(gcmHashContext *ghash, const unsigned char *buf, unsigned int count); SECStatus gcm_HashMult_sftw32(gcmHashContext *ghash, const unsigned char *buf, unsigned int count); -/* Stub definitions for the above *_hw functions, which shouldn't be - * used unless NSS_X86_OR_X64 is defined */ -#ifndef NSS_X86_OR_X64 -SECStatus -gcm_HashWrite_hw(gcmHashContext *ghash, unsigned char *outbuf) -{ - PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); - return SECFailure; -} - -SECStatus -gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf, - unsigned int count) -{ - PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); - return SECFailure; -} - -SECStatus -gcm_HashInit_hw(gcmHashContext *ghash) -{ - PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); - return SECFailure; -} - -SECStatus -gcm_HashZeroX_hw(gcmHashContext *ghash) -{ - PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); - return SECFailure; -} -#endif /* NSS_X86_OR_X64 */ - uint64_t get64(const unsigned char *bytes) { @@ -78,8 +46,6 @@ get64(const unsigned char *bytes) SECStatus gcmHash_InitContext(gcmHashContext *ghash, const unsigned char *H, PRBool sw) { - SECStatus rv = SECSuccess; - ghash->cLen = 0; ghash->bufLen = 0; PORT_Memset(ghash->counterBuf, 0, sizeof(ghash->counterBuf)); @@ -87,7 +53,17 @@ gcmHash_InitContext(gcmHashContext *ghash, const unsigned char *H, PRBool sw) ghash->h_low = get64(H + 8); ghash->h_high = get64(H); if (clmul_support() && !sw) { - rv = gcm_HashInit_hw(ghash); +#ifdef NSS_X86_OR_X64 + ghash->ghash_mul = gcm_HashMult_hw; + ghash->x = _mm_setzero_si128(); + /* MSVC requires __m64 to load epi64. */ + ghash->h = _mm_set_epi32(ghash->h_high >> 32, (uint32_t)ghash->h_high, + ghash->h_low >> 32, (uint32_t)ghash->h_low); + ghash->hw = PR_TRUE; +#else + PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); + return SECFailure; +#endif /* NSS_X86_OR_X64 */ } else { /* We fall back to the software implementation if we can't use / don't * want to use pclmul. */ @@ -99,7 +75,7 @@ gcmHash_InitContext(gcmHashContext *ghash, const unsigned char *H, PRBool sw) ghash->x_high = ghash->x_low = 0; ghash->hw = PR_FALSE; } - return rv; + return SECSuccess; } #ifdef HAVE_INT128_SUPPORT @@ -307,17 +283,102 @@ gcm_HashMult_sftw32(gcmHashContext *ghash, const unsigned char *buf, } #endif /* HAVE_INT128_SUPPORT */ +SECStatus +gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf, + unsigned int count) +{ +#ifdef NSS_X86_OR_X64 + size_t i; + pre_align __m128i z_high post_align; + pre_align __m128i z_low post_align; + pre_align __m128i C post_align; + pre_align __m128i D post_align; + pre_align __m128i E post_align; + pre_align __m128i F post_align; + pre_align __m128i bin post_align; + pre_align __m128i Ci post_align; + pre_align __m128i tmp post_align; + + for (i = 0; i < count; i++, buf += 16) { + bin = _mm_set_epi16(((uint16_t)buf[0] << 8) | buf[1], + ((uint16_t)buf[2] << 8) | buf[3], + ((uint16_t)buf[4] << 8) | buf[5], + ((uint16_t)buf[6] << 8) | buf[7], + ((uint16_t)buf[8] << 8) | buf[9], + ((uint16_t)buf[10] << 8) | buf[11], + ((uint16_t)buf[12] << 8) | buf[13], + ((uint16_t)buf[14] << 8) | buf[15]); + Ci = _mm_xor_si128(bin, ghash->x); + + /* Do binary mult ghash->X = Ci * ghash->H. */ + C = _mm_clmulepi64_si128(Ci, ghash->h, 0x00); + D = _mm_clmulepi64_si128(Ci, ghash->h, 0x11); + E = _mm_clmulepi64_si128(Ci, ghash->h, 0x01); + F = _mm_clmulepi64_si128(Ci, ghash->h, 0x10); + tmp = _mm_xor_si128(E, F); + z_high = _mm_xor_si128(tmp, _mm_slli_si128(D, 8)); + z_high = _mm_unpackhi_epi64(z_high, D); + z_low = _mm_xor_si128(_mm_slli_si128(tmp, 8), C); + z_low = _mm_unpackhi_epi64(_mm_slli_si128(C, 8), z_low); + + /* Shift one to the left (multiply by x) as gcm spec is stupid. */ + C = _mm_slli_si128(z_low, 8); + E = _mm_srli_epi64(C, 63); + D = _mm_slli_si128(z_high, 8); + F = _mm_srli_epi64(D, 63); + /* Carry over */ + C = _mm_srli_si128(z_low, 8); + D = _mm_srli_epi64(C, 63); + z_low = _mm_or_si128(_mm_slli_epi64(z_low, 1), E); + z_high = _mm_or_si128(_mm_or_si128(_mm_slli_epi64(z_high, 1), F), D); + + /* Reduce */ + C = _mm_slli_si128(z_low, 8); + /* D = z_low << 127 */ + D = _mm_slli_epi64(C, 63); + /* E = z_low << 126 */ + E = _mm_slli_epi64(C, 62); + /* F = z_low << 121 */ + F = _mm_slli_epi64(C, 57); + /* z_low ^= (z_low << 127) ^ (z_low << 126) ^ (z_low << 121); */ + z_low = _mm_xor_si128(_mm_xor_si128(_mm_xor_si128(z_low, D), E), F); + C = _mm_srli_si128(z_low, 8); + /* D = z_low >> 1 */ + D = _mm_slli_epi64(C, 63); + D = _mm_or_si128(_mm_srli_epi64(z_low, 1), D); + /* E = z_low >> 2 */ + E = _mm_slli_epi64(C, 62); + E = _mm_or_si128(_mm_srli_epi64(z_low, 2), E); + /* F = z_low >> 7 */ + F = _mm_slli_epi64(C, 57); + F = _mm_or_si128(_mm_srli_epi64(z_low, 7), F); + /* ghash->x ^= z_low ^ (z_low >> 1) ^ (z_low >> 2) ^ (z_low >> 7); */ + ghash->x = _mm_xor_si128(_mm_xor_si128( + _mm_xor_si128(_mm_xor_si128(z_high, z_low), D), E), + F); + } + return SECSuccess; +#else + PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); + return SECFailure; +#endif /* NSS_X86_OR_X64 */ +} + static SECStatus gcm_zeroX(gcmHashContext *ghash) { - SECStatus rv = SECSuccess; - if (ghash->hw) { - rv = gcm_HashZeroX_hw(ghash); +#ifdef NSS_X86_OR_X64 + ghash->x = _mm_setzero_si128(); + return SECSuccess; +#else + PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); + return SECFailure; +#endif /* NSS_X86_OR_X64 */ } ghash->x_high = ghash->x_low = 0; - return rv; + return SECSuccess; } /* @@ -442,10 +503,15 @@ gcmHash_Final(gcmHashContext *ghash, unsigned char *outbuf, } if (ghash->hw) { - rv = gcm_HashWrite_hw(ghash, T); - if (rv != SECSuccess) { - goto cleanup; - } +#ifdef NSS_X86_OR_X64 + uint64_t tmp_out[2]; + _mm_storeu_si128((__m128i *)tmp_out, ghash->x); + WRITE64(tmp_out[0], T + 8); + WRITE64(tmp_out[1], T); +#else + PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); + return SECFailure; +#endif /* NSS_X86_OR_X64 */ } else { WRITE64(ghash->x_low, T + 8); WRITE64(ghash->x_high, T); @@ -529,7 +595,14 @@ GCM_CreateContext(void *context, freeblCipherFunc cipher, if (gcm == NULL) { return NULL; } - ghash = PORT_ZNewAligned(gcmHashContext, 16, mem); + /* aligned_alloc is C11 so we have to do it the old way. */ + ghash = PORT_ZAlloc(sizeof(gcmHashContext) + 15); + if (ghash == NULL) { + PORT_SetError(SEC_ERROR_NO_MEMORY); + goto loser; + } + ghash->mem = ghash; + ghash = (gcmHashContext *)(((uintptr_t)ghash + 15) & ~(uintptr_t)0x0F); /* first plug in the ghash context */ gcm->ghash_context = ghash; |