Skip to content

Commit

Permalink
Rework RandomX tooling
Browse files Browse the repository at this point in the history 
- Move Fast/Light flag into the randomx state

now we only need the _fast vs _light methods on init

- Add randomx_info_nif and update lib/RandomX submodule

randomx_info_nif provides some info about a RandomX state -
currently only used in tests.

Updated lib/RandomX submodule exposts randomx dataset and
cache size values via -D build flags

- Change up how we handle DEBUG vs. non-DEBUG for ar_mine_randomx.erl

No longer us -define, instead switch behavior based on the
randomx State. This will let us test the non-DEBUG functionality
for some more code paths

- Add more ar_mine_randomx tests:

1. Test for a memory corruption issue in the decrypt_composite_nif
2. Tests for the NIF wrappers

- Remove the reencrypt_legacy_composit nif

  in preparation for increasing the randomx dataset for composite packing

- Remove obsolete c_src tests binary
- Remove ar_randomx_state
  • Loading branch information
@JamesPiechota
JamesPiechota authored and Lev Berman committed Sep 5, 2024
1 parent 739d652 commit 8a9ca76
Show file tree
Hide file tree
Showing 33 changed files with 705 additions and 1,332 deletions.
16 changes: 1 addition & 15 deletions apps/arweave/c_src/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ ERL_INTERFACE_LIB_DIR ?= $(shell erl -noshell -eval 'io:format("~ts", [code:lib_

C_SRC_DIR = $(CURDIR)
C_SRC_OUTPUT ?= $(CURDIR)/../priv/arweave.so
TESTS_DIR = $(CURDIR)/tests
TESTS_OUTPUT = $(TESTS_DIR)/tests

# System type and C compiler/flags.

Expand Down Expand Up @@ -77,14 +75,7 @@ link_verbose_0 = @echo " LD " $(@F);
link_verbose = $(link_verbose_$(V))

ALL_LIB_SOURCES := $(wildcard $(C_SRC_DIR)/*.c $(C_SRC_DIR)/*.cpp)
ENIF_SOURCES := $(wildcard $(C_SRC_DIR)/ar_*.c $(C_SRC_DIR)/ar_*.cpp)
C_SOURCES := $(filter-out $(ENIF_SOURCES), $(ALL_LIB_SOURCES))
TESTS_SOURCES = $(wildcard $(TESTS_DIR)/*.cpp)

ALL_LIB_OBJECTS = $(addsuffix .o, $(basename $(ALL_LIB_SOURCES)))
ENIF_OBJECTS = $(addsuffix .o, $(basename $(ENIF_SOURCES)))
C_OBJECTS = $(addsuffix .o, $(basename $(C_SOURCES)))
TESTS_OBJECTS = $(addsuffix .o, $(basename $(TESTS_SOURCES)))

COMPILE_C = $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c
COMPILE_CPP = $(cpp_verbose) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c
Expand All @@ -105,13 +96,8 @@ $(C_SRC_OUTPUT): $(ALL_LIB_OBJECTS)
%.o: %.cpp
$(COMPILE_CPP) $(OUTPUT_OPTION) $<

$(TESTS_OUTPUT): $(C_OBJECTS) $(TESTS_OBJECTS)
$(link_verbose) $(CXX) $(C_OBJECTS) $(TESTS_OBJECTS) $(LDFLAGS) $(LDLIBS) -o $(TESTS_OUTPUT)

tests: $(TESTS_OUTPUT)

clean:
@rm -f $(C_SRC_OUTPUT) $(TESTS_OUTPUT) $(ALL_LIB_OBJECTS) $(TESTS_OBJECTS)
@rm -f $(C_SRC_OUTPUT) $(ALL_LIB_OBJECTS)



Expand Down
211 changes: 68 additions & 143 deletions apps/arweave/c_src/ar_mine_randomx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,9 @@ ErlNifResourceType* vdfRandomxVmType;
#include "ar_mine_vdf.h"

static ErlNifFunc nif_funcs[] = {
{"init_fast_nif", 4, init_fast_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"init_light_nif", 3, init_light_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"hash_fast_nif", 5, hash_fast_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"hash_light_nif", 5, hash_light_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_info_nif", 1, randomx_info_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"init_randomx_nif", 5, init_randomx_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"hash_nif", 5, randomx_hash_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_encrypt_chunk_nif", 7, randomx_encrypt_chunk_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_decrypt_chunk_nif", 8, randomx_decrypt_chunk_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_reencrypt_chunk_nif", 10, randomx_reencrypt_chunk_nif,
Expand All @@ -28,11 +27,8 @@ static ErlNifFunc nif_funcs[] = {
ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_decrypt_composite_sub_chunk_nif", 10, randomx_decrypt_composite_sub_chunk_nif,
ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_reencrypt_legacy_to_composite_chunk_nif", 11,
randomx_reencrypt_legacy_to_composite_chunk_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_reencrypt_composite_to_composite_chunk_nif", 13,
randomx_reencrypt_composite_to_composite_chunk_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"release_state_nif", 1, release_state_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"randomx_reencrypt_composite_chunk_nif", 13,
randomx_reencrypt_composite_chunk_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"vdf_sha2_nif", 5, vdf_sha2_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
{"vdf_parallel_sha_verify_with_reset_nif", 10, vdf_parallel_sha_verify_with_reset_nif,
ERL_NIF_DIRTY_JOB_CPU_BOUND}
Expand Down Expand Up @@ -75,51 +71,88 @@ static void release_randomx(struct state *statePtr)
statePtr->isRandomxReleased = 1;
}

static ERL_NIF_TERM init_fast_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
static ERL_NIF_TERM randomx_info_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
{
return init(envPtr, argc, argv, HASHING_MODE_FAST);
struct state* statePtr;
unsigned int datasetSize;
hashing_mode hashingMode;
ERL_NIF_TERM hashingModeTerm;

if (argc != 1) {
return enif_make_badarg(envPtr);
}
if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) {
return error(envPtr, "failed to read state");
}

hashingMode = statePtr->mode;

if (hashingMode == HASHING_MODE_FAST) {
if (statePtr->datasetPtr == NULL) {
return error(envPtr, "dataset is not initialized for fast hashing mode");
}
if (statePtr->cachePtr != NULL) {
return error(envPtr, "cache is initialized for fast hashing mode");
}
datasetSize = randomx_dataset_item_count();
hashingModeTerm = enif_make_atom(envPtr, "fast");
} else if (hashingMode == HASHING_MODE_LIGHT) {
if (statePtr->datasetPtr != NULL) {
return error(envPtr, "dataset is initialized for light hashing mode");
}
if (statePtr->cachePtr == NULL) {
return error(envPtr, "cache is not initialized for light hashing mode");
}
datasetSize = 0;
hashingModeTerm = enif_make_atom(envPtr, "light");
} else {
return error(envPtr, "invalid hashing mode");
}


return ok_tuple2(envPtr, hashingModeTerm, enif_make_uint(envPtr, datasetSize));
}

static ERL_NIF_TERM init_light_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])

static ERL_NIF_TERM init_randomx_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
{
return init(envPtr, argc, argv, HASHING_MODE_LIGHT);
return init(envPtr, argc, argv);
}

static ERL_NIF_TERM init(
ErlNifEnv* envPtr,
int argc,
const ERL_NIF_TERM argv[],
hashing_mode mode
const ERL_NIF_TERM argv[]
) {
ErlNifBinary key;
hashing_mode mode;
struct state *statePtr;
ERL_NIF_TERM resource;
unsigned int numWorkers;
int jitEnabled, largePagesEnabled;
randomx_flags flags;

if (mode == HASHING_MODE_FAST && argc != 4) {
return enif_make_badarg(envPtr);
} else if (mode == HASHING_MODE_LIGHT && argc != 3) {
if (!enif_inspect_binary(envPtr, argv[0], &key)) {
return enif_make_badarg(envPtr);
}
if (!enif_inspect_binary(envPtr, argv[0], &key)) {
if (!enif_get_int(envPtr, argv[1], &mode)) {
return enif_make_badarg(envPtr);
}
if (mode == HASHING_MODE_FAST && !enif_get_uint(envPtr, argv[3], &numWorkers)) {
if (!enif_get_int(envPtr, argv[2], &jitEnabled)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[1], &jitEnabled)) {
if (!enif_get_int(envPtr, argv[3], &largePagesEnabled)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[2], &largePagesEnabled)) {
if (!enif_get_uint(envPtr, argv[4], &numWorkers)) {
return enif_make_badarg(envPtr);
}

statePtr = enif_alloc_resource(stateType, sizeof(struct state));
statePtr->cachePtr = NULL;
statePtr->datasetPtr = NULL;
statePtr->isRandomxReleased = 0;
statePtr->mode = mode;

statePtr->lockPtr = enif_rwlock_create("state_rw_lock");
if (statePtr->lockPtr == NULL) {
Expand Down Expand Up @@ -256,16 +289,6 @@ static ERL_NIF_TERM init_failed(ErlNifEnv *envPtr, struct state *statePtr, const
return error(envPtr, reason);
}

static ERL_NIF_TERM hash_fast_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
{
return randomx_hash_nif(envPtr, argc, argv, HASHING_MODE_FAST);
}

static ERL_NIF_TERM hash_light_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
{
return randomx_hash_nif(envPtr, argc, argv, HASHING_MODE_LIGHT);
}

static randomx_vm* create_vm(struct state* statePtr,
int fullMemEnabled, int jitEnabled, int largePagesEnabled, int hardwareAESEnabled,
int* isRandomxReleased) {
Expand Down Expand Up @@ -461,8 +484,7 @@ static ERL_NIF_TERM decrypt_composite_chunk(ErlNifEnv* envPtr,
static ERL_NIF_TERM randomx_hash_nif(
ErlNifEnv* envPtr,
int argc,
const ERL_NIF_TERM argv[],
hashing_mode hashingMode
const ERL_NIF_TERM argv[]
) {
int jitEnabled, largePagesEnabled, hardwareAESEnabled;
unsigned char hashPtr[RANDOMX_HASH_SIZE];
Expand All @@ -489,7 +511,7 @@ static ERL_NIF_TERM randomx_hash_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, (hashingMode == HASHING_MODE_FAST), jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST), jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
return error(envPtr, "state has been released");
Expand Down Expand Up @@ -542,7 +564,8 @@ static ERL_NIF_TERM randomx_encrypt_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1, jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
return error(envPtr, "state has been released");
Expand Down Expand Up @@ -597,7 +620,8 @@ static ERL_NIF_TERM randomx_decrypt_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1, jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
return error(envPtr, "state has been released");
Expand Down Expand Up @@ -667,7 +691,8 @@ static ERL_NIF_TERM randomx_reencrypt_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1, jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
return error(envPtr, "state has been released");
Expand Down Expand Up @@ -746,7 +771,7 @@ static ERL_NIF_TERM randomx_encrypt_composite_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1,
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
Expand Down Expand Up @@ -822,7 +847,7 @@ static ERL_NIF_TERM randomx_decrypt_composite_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1,
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
Expand Down Expand Up @@ -904,7 +929,7 @@ static ERL_NIF_TERM randomx_decrypt_composite_sub_chunk_nif(
uint32_t subChunkSize = outChunkLen;
unsigned char key[PACKING_KEY_SIZE];

randomx_vm *vmPtr = create_vm(statePtr, 1,
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
Expand Down Expand Up @@ -944,89 +969,7 @@ static ERL_NIF_TERM randomx_decrypt_composite_sub_chunk_nif(
return ok_tuple(envPtr, decryptedSubChunkTerm);
}

static ERL_NIF_TERM randomx_reencrypt_legacy_to_composite_chunk_nif(
ErlNifEnv* envPtr,
int argc,
const ERL_NIF_TERM argv[]
) {
int decryptRandomxRoundCount, encryptRandomxRoundCount;
int jitEnabled, largePagesEnabled, hardwareAESEnabled;
int subChunkCount, iterations;
struct state* statePtr;
ErlNifBinary decryptKey;
ErlNifBinary encryptKey;
ErlNifBinary inputChunk;

if (argc != 11) {
return enif_make_badarg(envPtr);
}
if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) {
return error(envPtr, "failed to read state");
}
if (!enif_inspect_binary(envPtr, argv[1], &decryptKey)) {
return enif_make_badarg(envPtr);
}
if (!enif_inspect_binary(envPtr, argv[2], &encryptKey)) {
return enif_make_badarg(envPtr);
}
if (!enif_inspect_binary(envPtr, argv[3], &inputChunk) ||
inputChunk.size != MAX_CHUNK_SIZE) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[4], &jitEnabled)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[5], &largePagesEnabled)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[6], &hardwareAESEnabled)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[7], &decryptRandomxRoundCount)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[8], &encryptRandomxRoundCount)) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[9], &iterations) ||
iterations < 1) {
return enif_make_badarg(envPtr);
}
if (!enif_get_int(envPtr, argv[10], &subChunkCount) ||
subChunkCount < 1 ||
MAX_CHUNK_SIZE % subChunkCount != 0 ||
(MAX_CHUNK_SIZE / subChunkCount) % 64 != 0 ||
subChunkCount > (MAX_CHUNK_SIZE / 64)) {
return enif_make_badarg(envPtr);
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1,
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
return error(envPtr, "state has been released");
}
return error(envPtr, "randomx_create_vm failed");
}

unsigned char decryptedChunk[MAX_CHUNK_SIZE];
ERL_NIF_TERM decryptedChunkTerm = decrypt_chunk(envPtr, vmPtr,
decryptKey.data, decryptKey.size, inputChunk.data, inputChunk.size,
decryptedChunk, inputChunk.size, decryptRandomxRoundCount);
ErlNifBinary decryptedChunkBin;
if (!enif_inspect_binary(envPtr, decryptedChunkTerm, &decryptedChunkBin)) {
destroy_vm(statePtr, vmPtr);
return enif_make_badarg(envPtr);
}
ERL_NIF_TERM reencryptedChunkTerm = encrypt_composite_chunk(envPtr, vmPtr, &encryptKey,
&decryptedChunkBin, subChunkCount, iterations, encryptRandomxRoundCount,
jitEnabled, largePagesEnabled, hardwareAESEnabled);
destroy_vm(statePtr, vmPtr);
return ok_tuple2(envPtr, reencryptedChunkTerm, decryptedChunkTerm);
}

static ERL_NIF_TERM randomx_reencrypt_composite_to_composite_chunk_nif(
static ERL_NIF_TERM randomx_reencrypt_composite_chunk_nif(
ErlNifEnv* envPtr,
int argc,
const ERL_NIF_TERM argv[]
Expand Down Expand Up @@ -1096,7 +1039,7 @@ static ERL_NIF_TERM randomx_reencrypt_composite_to_composite_chunk_nif(
}

int isRandomxReleased;
randomx_vm *vmPtr = create_vm(statePtr, 1,
randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST),
jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased);
if (vmPtr == NULL) {
if (isRandomxReleased != 0) {
Expand Down Expand Up @@ -1152,24 +1095,6 @@ static ERL_NIF_TERM randomx_reencrypt_composite_to_composite_chunk_nif(
return ok_tuple2(envPtr, reencryptedChunkTerm, decryptedChunkTerm);
}

static ERL_NIF_TERM release_state_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[])
{
struct state* statePtr;

if (argc != 1) {
return enif_make_badarg(envPtr);
}
if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) {
return error(envPtr, "failed to read state");
}
if (enif_rwlock_tryrwlock(statePtr->lockPtr) != 0) {
return error(envPtr, "failed to acquire the state lock, the state is being used");
}
release_randomx(statePtr);
enif_rwlock_rwunlock(statePtr->lockPtr);
return enif_make_atom(envPtr, "ok");
}

// Utility functions.

static ERL_NIF_TERM solution_tuple(ErlNifEnv* envPtr, ERL_NIF_TERM hashTerm) {
Expand Down
Loading

0 comments on commit 8a9ca76

Please sign in to comment.