From 6f43ad9498c3de32cb7c9a8a017a656a97c79e5d Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Tue, 17 Jan 2017 19:57:04 -0500 Subject: [PATCH] unifies cryptonight fetch/no-prefetch functions --- ...ryptonight_aesni.c => cryptonight_aesni.h} | 118 +++++++----------- ...onight_common.c => cryptonight_common.cpp} | 33 +++-- 2 files changed, 68 insertions(+), 83 deletions(-) rename crypto/{cryptonight_aesni.c => cryptonight_aesni.h} (78%) rename crypto/{cryptonight_common.c => cryptonight_common.cpp} (77%) diff --git a/crypto/cryptonight_aesni.c b/crypto/cryptonight_aesni.h similarity index 78% rename from crypto/cryptonight_aesni.c rename to crypto/cryptonight_aesni.h index 0351931..4bf099e 100644 --- a/crypto/cryptonight_aesni.c +++ b/crypto/cryptonight_aesni.h @@ -13,6 +13,7 @@ * along with this program. If not, see . * */ +#pragma once #include "cryptonight.h" #include @@ -36,9 +37,12 @@ static inline uint64_t _umul128(uint64_t a, uint64_t b, uint64_t* hi) #error You are trying to do a 32-bit build. This will all end in tears. I know it. #endif -void keccak(const uint8_t *in, int inlen, uint8_t *md, int mdlen); -void keccakf(uint64_t st[25], int rounds); -extern void(*const extra_hashes[4])(const void *, size_t, char *); +extern "C" +{ + void keccak(const uint8_t *in, int inlen, uint8_t *md, int mdlen); + void keccakf(uint64_t st[25], int rounds); + extern void(*const extra_hashes[4])(const void *, size_t, char *); +} // This will shift and xor tmp1 into itself as 4 32-bit vals such as // sl_xor(a1 a2 a3 a4) = a1 (a2^a1) (a3^a2^a1) (a4^a3^a2^a1) @@ -125,6 +129,7 @@ static inline void aes_round(__m128i key, __m128i* x0, __m128i* x1, __m128i* x2, *x7 = _mm_aesenc_si128(*x7, key); } +template void cn_explode_scratchpad(const __m128i* input, __m128i* output) { // This is more than we have registers, compiler will assign 2 keys on the stack @@ -142,7 +147,7 @@ void cn_explode_scratchpad(const __m128i* input, __m128i* output) xin6 = _mm_load_si128(input + 10); xin7 = _mm_load_si128(input + 11); - for (size_t i = 0; i < MEMORY / sizeof(__m128i); i += 8) + for (size_t i = 0; i < MEM / sizeof(__m128i); i += 8) { aes_round(k0, &xin0, &xin1, &xin2, &xin3, &xin4, &xin5, &xin6, &xin7); aes_round(k1, &xin0, &xin1, &xin2, &xin3, &xin4, &xin5, &xin6, &xin7); @@ -159,15 +164,16 @@ void cn_explode_scratchpad(const __m128i* input, __m128i* output) _mm_store_si128(output + i + 1, xin1); _mm_store_si128(output + i + 2, xin2); _mm_store_si128(output + i + 3, xin3); - _mm_prefetch(output + i + 0, _MM_HINT_T2); + _mm_prefetch((const char*)output + i + 0, _MM_HINT_T2); _mm_store_si128(output + i + 4, xin4); _mm_store_si128(output + i + 5, xin5); _mm_store_si128(output + i + 6, xin6); _mm_store_si128(output + i + 7, xin7); - _mm_prefetch(output + i + 4, _MM_HINT_T2); + _mm_prefetch((const char*)output + i + 4, _MM_HINT_T2); } } +template void cn_implode_scratchpad(const __m128i* input, __m128i* output) { // This is more than we have registers, compiler will assign 2 keys on the stack @@ -185,14 +191,14 @@ void cn_implode_scratchpad(const __m128i* input, __m128i* output) xout6 = _mm_load_si128(output + 10); xout7 = _mm_load_si128(output + 11); - for (size_t i = 0; i < MEMORY / sizeof(__m128i); i += 8) + for (size_t i = 0; i < MEM / sizeof(__m128i); i += 8) { - _mm_prefetch(input + i + 0, _MM_HINT_NTA); + _mm_prefetch((const char*)input + i + 0, _MM_HINT_NTA); xout0 = _mm_xor_si128(_mm_load_si128(input + i + 0), xout0); xout1 = _mm_xor_si128(_mm_load_si128(input + i + 1), xout1); xout2 = _mm_xor_si128(_mm_load_si128(input + i + 2), xout2); xout3 = _mm_xor_si128(_mm_load_si128(input + i + 3), xout3); - _mm_prefetch(input + i + 4, _MM_HINT_NTA); + _mm_prefetch((const char*)input + i + 4, _MM_HINT_NTA); xout4 = _mm_xor_si128(_mm_load_si128(input + i + 4), xout4); xout5 = _mm_xor_si128(_mm_load_si128(input + i + 5), xout5); xout6 = _mm_xor_si128(_mm_load_si128(input + i + 6), xout6); @@ -220,12 +226,13 @@ void cn_implode_scratchpad(const __m128i* input, __m128i* output) _mm_store_si128(output + 11, xout7); } -void cryptonight_hash_ctx(const void* input, size_t len, void* output, cryptonight_ctx* ctx0) +template +void cryptonight_hash(const void* input, size_t len, void* output, cryptonight_ctx* ctx0) { keccak((const uint8_t *)input, len, ctx0->hash_state, 200); // Optim - 99% time boundary - cn_explode_scratchpad((__m128i*)ctx0->hash_state, (__m128i*)ctx0->long_state); + cn_explode_scratchpad((__m128i*)ctx0->hash_state, (__m128i*)ctx0->long_state); uint8_t* l0 = ctx0->long_state; uint64_t* h0 = (uint64_t*)ctx0->hash_state; @@ -237,7 +244,7 @@ void cryptonight_hash_ctx(const void* input, size_t len, void* output, cryptonig uint64_t idx0 = h0[0] ^ h0[4]; // Optim - 90% time boundary - for(size_t i = 0; i < 0x80000; i++) + for(size_t i = 0; i < ITERATIONS; i++) { __m128i cx; cx = _mm_load_si128((__m128i *)&l0[idx0 & 0x1FFFF0]); @@ -245,7 +252,8 @@ void cryptonight_hash_ctx(const void* input, size_t len, void* output, cryptonig _mm_store_si128((__m128i *)&l0[idx0 & 0x1FFFF0], _mm_xor_si128(bx0, cx)); idx0 = _mm_cvtsi128_si64(cx); bx0 = cx; - _mm_prefetch(&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); uint64_t hi, lo, cl, ch; cl = ((uint64_t*)&l0[idx0 & 0x1FFFF0])[0]; @@ -258,77 +266,31 @@ void cryptonight_hash_ctx(const void* input, size_t len, void* output, cryptonig ah0 ^= ch; al0 ^= cl; idx0 = al0; - _mm_prefetch(&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); } // Optim - 90% time boundary - cn_implode_scratchpad((__m128i*)ctx0->long_state, (__m128i*)ctx0->hash_state); + cn_implode_scratchpad((__m128i*)ctx0->long_state, (__m128i*)ctx0->hash_state); // Optim - 99% time boundary keccakf((uint64_t*)ctx0->hash_state, 24); - extra_hashes[ctx0->hash_state[0] & 3](ctx0->hash_state, 200, output); -} - -void cryptonight_hash_ctx_np(const void* input, size_t len, void* output, cryptonight_ctx* ctx0) -{ - keccak((const uint8_t *)input, len, ctx0->hash_state, 200); - - // Optim - 99% time boundary - cn_explode_scratchpad((__m128i*)ctx0->hash_state, (__m128i*)ctx0->long_state); - - uint8_t* l0 = ctx0->long_state; - uint64_t* h0 = (uint64_t*)ctx0->hash_state; - - uint64_t al0 = h0[0] ^ h0[4]; - uint64_t ah0 = h0[1] ^ h0[5]; - __m128i bx0 = _mm_set_epi64x(h0[3] ^ h0[7], h0[2] ^ h0[6]); - - uint64_t idx0 = h0[0] ^ h0[4]; - - // Optim - 90% time boundary - for(size_t i = 0; i < 0x80000; i++) - { - __m128i cx; - cx = _mm_load_si128((__m128i *)&l0[idx0 & 0x1FFFF0]); - cx = _mm_aesenc_si128(cx, _mm_set_epi64x(ah0, al0)); - _mm_store_si128((__m128i *)&l0[idx0 & 0x1FFFF0], _mm_xor_si128(bx0, cx)); - idx0 = _mm_cvtsi128_si64(cx); - bx0 = cx; - - uint64_t hi, lo, cl, ch; - cl = ((uint64_t*)&l0[idx0 & 0x1FFFF0])[0]; - ch = ((uint64_t*)&l0[idx0 & 0x1FFFF0])[1]; - lo = _umul128(idx0, cl, &hi); - al0 += hi; - ah0 += lo; - ((uint64_t*)&l0[idx0 & 0x1FFFF0])[0] = al0; - ((uint64_t*)&l0[idx0 & 0x1FFFF0])[1] = ah0; - ah0 ^= ch; - al0 ^= cl; - idx0 = al0; - } - - // Optim - 90% time boundary - cn_implode_scratchpad((__m128i*)ctx0->long_state, (__m128i*)ctx0->hash_state); - - // Optim - 99% time boundary - - keccakf((uint64_t*)ctx0->hash_state, 24); - extra_hashes[ctx0->hash_state[0] & 3](ctx0->hash_state, 200, output); + extra_hashes[ctx0->hash_state[0] & 3](ctx0->hash_state, 200, (char*)output); } // This lovely creation will do 2 cn hashes at a time. We have plenty of space on silicon // to fit temporary vars for two contexts. Function will read len*2 from input and write 64 bytes to output // We are still limited by L3 cache, so doubling will only work with CPUs where we have more than 2MB to core (Xeons) -void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cryptonight_ctx* __restrict ctx0, cryptonight_ctx* __restrict ctx1) +template +void cryptonight_double_hash(const void* input, size_t len, void* output, cryptonight_ctx* __restrict ctx0, cryptonight_ctx* __restrict ctx1) { keccak((const uint8_t *)input, len, ctx0->hash_state, 200); keccak((const uint8_t *)input+len, len, ctx1->hash_state, 200); // Optim - 99% time boundary - cn_explode_scratchpad((__m128i*)ctx0->hash_state, (__m128i*)ctx0->long_state); - cn_explode_scratchpad((__m128i*)ctx1->hash_state, (__m128i*)ctx1->long_state); + cn_explode_scratchpad((__m128i*)ctx0->hash_state, (__m128i*)ctx0->long_state); + cn_explode_scratchpad((__m128i*)ctx1->hash_state, (__m128i*)ctx1->long_state); uint8_t* l0 = ctx0->long_state; uint64_t* h0 = (uint64_t*)ctx0->hash_state; @@ -344,7 +306,7 @@ void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cr uint64_t idx1 = h1[0] ^ h1[4]; // Optim - 90% time boundary - for (size_t i = 0; i < 0x80000; i++) + for (size_t i = 0; i < ITERATIONS; i++) { __m128i cx; cx = _mm_load_si128((__m128i *)&l0[idx0 & 0x1FFFF0]); @@ -352,14 +314,16 @@ void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cr _mm_store_si128((__m128i *)&l0[idx0 & 0x1FFFF0], _mm_xor_si128(bx0, cx)); idx0 = _mm_cvtsi128_si64(cx); bx0 = cx; - _mm_prefetch(&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); cx = _mm_load_si128((__m128i *)&l1[idx1 & 0x1FFFF0]); cx = _mm_aesenc_si128(cx, ax1); _mm_store_si128((__m128i *)&l1[idx1 & 0x1FFFF0], _mm_xor_si128(bx1, cx)); idx1 = _mm_cvtsi128_si64(cx); bx1 = cx; - _mm_prefetch(&l1[idx1 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l1[idx1 & 0x1FFFF0], _MM_HINT_T0); uint64_t hi, lo; cx = _mm_load_si128((__m128i *)&l0[idx0 & 0x1FFFF0]); @@ -368,7 +332,8 @@ void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cr _mm_store_si128((__m128i*)&l0[idx0 & 0x1FFFF0], ax0); ax0 = _mm_xor_si128(ax0, cx); idx0 = _mm_cvtsi128_si64(ax0); - _mm_prefetch(&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l0[idx0 & 0x1FFFF0], _MM_HINT_T0); cx = _mm_load_si128((__m128i *)&l1[idx1 & 0x1FFFF0]); lo = _umul128(idx1, _mm_cvtsi128_si64(cx), &hi); @@ -376,17 +341,18 @@ void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cr _mm_store_si128((__m128i*)&l1[idx1 & 0x1FFFF0], ax1); ax1 = _mm_xor_si128(ax1, cx); idx1 = _mm_cvtsi128_si64(ax1); - _mm_prefetch(&l1[idx1 & 0x1FFFF0], _MM_HINT_T0); + if(PREFETCH) + _mm_prefetch((const char*)&l1[idx1 & 0x1FFFF0], _MM_HINT_T0); } // Optim - 90% time boundary - cn_implode_scratchpad((__m128i*)ctx0->long_state, (__m128i*)ctx0->hash_state); - cn_implode_scratchpad((__m128i*)ctx1->long_state, (__m128i*)ctx1->hash_state); + cn_implode_scratchpad((__m128i*)ctx0->long_state, (__m128i*)ctx0->hash_state); + cn_implode_scratchpad((__m128i*)ctx1->long_state, (__m128i*)ctx1->hash_state); // Optim - 99% time boundary keccakf((uint64_t*)ctx0->hash_state, 24); - extra_hashes[ctx0->hash_state[0] & 3](ctx0->hash_state, 200, output); + extra_hashes[ctx0->hash_state[0] & 3](ctx0->hash_state, 200, (char*)output); keccakf((uint64_t*)ctx1->hash_state, 24); extra_hashes[ctx1->hash_state[0] & 3](ctx1->hash_state, 200, (char*)output + 32); -} +} \ No newline at end of file diff --git a/crypto/cryptonight_common.c b/crypto/cryptonight_common.cpp similarity index 77% rename from crypto/cryptonight_common.c rename to crypto/cryptonight_common.cpp index 3e04b15..8a89d5d 100644 --- a/crypto/cryptonight_common.c +++ b/crypto/cryptonight_common.cpp @@ -13,11 +13,15 @@ * along with this program. If not, see . */ +extern "C" +{ #include "c_groestl.h" #include "c_blake256.h" #include "c_jh.h" #include "c_skein.h" +} #include "cryptonight.h" +#include "cryptonight_aesni.h" #include #include @@ -36,19 +40,19 @@ #endif // _WIN32 void do_blake_hash(const void* input, size_t len, char* output) { - blake256_hash((uint8_t*)output, input, len); + blake256_hash((uint8_t*)output, (const uint8_t*)input, len); } void do_groestl_hash(const void* input, size_t len, char* output) { - groestl(input, len * 8, (uint8_t*)output); + groestl((const uint8_t*)input, len * 8, (uint8_t*)output); } void do_jh_hash(const void* input, size_t len, char* output) { - jh_hash(32 * 8, input, 8 * len, (uint8_t*)output); + jh_hash(32 * 8, (const uint8_t*)input, 8 * len, (uint8_t*)output); } void do_skein_hash(const void* input, size_t len, char* output) { - skein_hash(8 * 32, input, 8 * len, (uint8_t*)output); + skein_hash(8 * 32, (const uint8_t*)input, 8 * len, (uint8_t*)output); } void (* const extra_hashes[4])(const void *, size_t, char *) = {do_blake_hash, do_groestl_hash, do_jh_hash, do_skein_hash}; @@ -94,11 +98,11 @@ size_t cryptonight_init(size_t use_fast_mem, size_t use_mlock, alloc_msg* msg) cryptonight_ctx* cryptonight_alloc_ctx(size_t use_fast_mem, size_t use_mlock, alloc_msg* msg) { - cryptonight_ctx* ptr = _mm_malloc(sizeof(cryptonight_ctx), 4096); + cryptonight_ctx* ptr = (cryptonight_ctx*)_mm_malloc(sizeof(cryptonight_ctx), 4096); if(use_fast_mem == 0) { - ptr->long_state = _mm_malloc(MEMORY, 4096); + ptr->long_state = (uint8_t*)_mm_malloc(MEMORY, 4096); ptr->ctx_info[0] = 0; ptr->ctx_info[1] = 0; return ptr; @@ -110,7 +114,7 @@ cryptonight_ctx* cryptonight_alloc_ctx(size_t use_fast_mem, size_t use_mlock, al if(MEMORY > iLargePageMin) iLargePageMin *= 2; - ptr->long_state = (cryptonight_ctx*)VirtualAlloc(NULL, iLargePageMin, + ptr->long_state = (uint8_t*)VirtualAlloc(NULL, iLargePageMin, MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE); if(ptr->long_state == NULL) @@ -167,3 +171,18 @@ void cryptonight_free_ctx(cryptonight_ctx* ctx) _mm_free(ctx); } + +void cryptonight_hash_ctx(const void* input, size_t len, void* output, cryptonight_ctx* ctx) +{ + cryptonight_hash<0x8000, MEMORY, true>(input, len, output, ctx); +} + +void cryptonight_hash_ctx_np(const void* input, size_t len, void* output, cryptonight_ctx* ctx) +{ + cryptonight_hash<0x8000, MEMORY, false>(input, len, output, ctx); +} + +void cryptonight_double_hash_ctx(const void* input, size_t len, void* output, cryptonight_ctx* __restrict ctx0, cryptonight_ctx* __restrict ctx1) +{ + cryptonight_double_hash<0x8000, MEMORY, false>(input, len, output, ctx0, ctx1); +} \ No newline at end of file