From b5af96f068fe9337f311aa88be5a250260134e45 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 28 Nov 2024 09:47:59 +0100 Subject: [PATCH 01/44] Move to version 2.0.6 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7c8f2b1a7..c502e23ac 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ # # If NIM_COMMIT is set to "nimbusbuild", this will use the # version pinned by nimbus-build-system. -PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21 - +# PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21 +PINNED_NIM_VERSION := v2.0.6 ifeq ($(NIM_COMMIT),) NIM_COMMIT := $(PINNED_NIM_VERSION) else ifeq ($(NIM_COMMIT),pinned) From ca2b63c3b1dc5ca77ccf7f9b371e547a5aa69547 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 28 Nov 2024 09:48:12 +0100 Subject: [PATCH 02/44] Update nim-confutils submodule to latest version --- vendor/nim-confutils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 2028b4160..cb858a27f 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 2028b41602b3abf7c9bf450744efde7b296707a2 +Subproject commit cb858a27f4347be949d10ed74b58713d687936d2 From ef3274b2902e426dc92cce11feb63e6b916e63c6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:25:38 +0100 Subject: [PATCH 03/44] Update dependencies --- vendor/asynctest | 2 +- vendor/codex-contracts-eth | 2 +- vendor/codex-storage-proofs-circuits | 2 +- vendor/combparser | 2 +- vendor/constantine | 2 +- vendor/nim-bearssl | 2 +- vendor/nim-blscurve | 2 +- vendor/nim-chronicles | 2 +- vendor/nim-chronos | 2 +- vendor/nim-circom-compat | 2 +- vendor/nim-codex-dht | 2 +- vendor/nim-contract-abi | 2 +- vendor/nim-datastore | 2 +- vendor/nim-eth | 2 +- vendor/nim-ethers | 2 +- vendor/nim-faststreams | 2 +- vendor/nim-http-utils | 2 +- vendor/nim-json-rpc | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-leopard | 2 +- vendor/nim-libbacktrace | 2 +- vendor/nim-libp2p | 2 +- vendor/nim-metrics | 2 +- vendor/nim-nat-traversal | 2 +- vendor/nim-nitro | 2 +- vendor/nim-poseidon2 | 2 +- vendor/nim-presto | 2 +- vendor/nim-protobuf-serialization | 2 +- vendor/nim-results | 2 +- vendor/nim-serde | 2 +- vendor/nim-serialization | 2 +- vendor/nim-sqlite3-abi | 2 +- vendor/nim-stew | 2 +- vendor/nim-taskpools | 2 +- vendor/nim-testutils | 2 +- vendor/nim-toml-serialization | 2 +- vendor/nim-unittest2 | 2 +- vendor/nim-websock | 2 +- vendor/nim-zlib | 2 +- vendor/nimbus-build-system | 2 +- vendor/nimcrypto | 2 +- vendor/npeg | 2 +- vendor/stint | 2 +- vendor/upraises | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) diff --git a/vendor/asynctest b/vendor/asynctest index 8e2f4e73b..5154c0d79 160000 --- a/vendor/asynctest +++ b/vendor/asynctest @@ -1 +1 @@ -Subproject commit 8e2f4e73b97123be0f0041c129942b32df23ecb1 +Subproject commit 5154c0d79dd8bb086ab418cc659e923330ac24f2 diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 945f6008c..06f9f56cd 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 945f6008c8817abd7ca43a40368d33bb1e014c14 +Subproject commit 06f9f56cd27f3eba5fee88a4cdf20668f3d9ce43 diff --git a/vendor/codex-storage-proofs-circuits b/vendor/codex-storage-proofs-circuits index c03b43221..ac8d36675 160000 --- a/vendor/codex-storage-proofs-circuits +++ b/vendor/codex-storage-proofs-circuits @@ -1 +1 @@ -Subproject commit c03b43221d68e34bd5015a4e4ee1a0ad3299f8ef +Subproject commit ac8d3667526862458b162bee71dd5dcf6170c209 diff --git a/vendor/combparser b/vendor/combparser index ba4464c00..e582c436e 160000 --- a/vendor/combparser +++ b/vendor/combparser @@ -1 +1 @@ -Subproject commit ba4464c005d7617c008e2ed2ebc1ba52feb469c6 +Subproject commit e582c436e8750b60253370fd77960509d36e3738 diff --git a/vendor/constantine b/vendor/constantine index 8367d7d19..bc3845aa4 160000 --- a/vendor/constantine +++ b/vendor/constantine @@ -1 +1 @@ -Subproject commit 8367d7d19cdbba874aab961b70d272e742184c37 +Subproject commit bc3845aa492b52f7fef047503b1592e830d1a774 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl index 99fcb3405..667b40440 160000 --- a/vendor/nim-bearssl +++ b/vendor/nim-bearssl @@ -1 +1 @@ -Subproject commit 99fcb3405c55b27cfffbf60f5368c55da7346f23 +Subproject commit 667b40440a53a58e9f922e29e20818720c62d9ac diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve index 48d8668c5..de2d3c792 160000 --- a/vendor/nim-blscurve +++ b/vendor/nim-blscurve @@ -1 +1 @@ -Subproject commit 48d8668c5a9a350d3a7ee0c3713ef9a11980a40d +Subproject commit de2d3c79264bba18dbea469c8c5c4b3bb3c8bc55 diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index c9c8e58ec..81a4a7a36 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit c9c8e58ec3f89b655a046c485f622f9021c68b61 +Subproject commit 81a4a7a360c78be9c80c8f735c76b6d4a1517304 diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 035ae11ba..c04576d82 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 035ae11ba92369e7722e649db597e79134fd06b9 +Subproject commit c04576d829b8a0a1b12baaa8bc92037501b3a4a0 diff --git a/vendor/nim-circom-compat b/vendor/nim-circom-compat index e710e4c33..d3fb90394 160000 --- a/vendor/nim-circom-compat +++ b/vendor/nim-circom-compat @@ -1 +1 @@ -Subproject commit e710e4c333f367353aaa1ee82a55a47326b63a65 +Subproject commit d3fb903945c3895f28a2e50685745e0a9762ece5 diff --git a/vendor/nim-codex-dht b/vendor/nim-codex-dht index 63822e835..5f22be042 160000 --- a/vendor/nim-codex-dht +++ b/vendor/nim-codex-dht @@ -1 +1 @@ -Subproject commit 63822e83561ea1c6396d0f3eca583b038f5d44c6 +Subproject commit 5f22be04208d8ead97eed0af8af47033e1211f23 diff --git a/vendor/nim-contract-abi b/vendor/nim-contract-abi index 61f8f59b3..363f8b485 160000 --- a/vendor/nim-contract-abi +++ b/vendor/nim-contract-abi @@ -1 +1 @@ -Subproject commit 61f8f59b3917d8e27c6eb4330a6d8cf428e98b2d +Subproject commit 363f8b4852b05c089b1e35ef3a0a882d51c996bc diff --git a/vendor/nim-datastore b/vendor/nim-datastore index 3ab6b84a6..b5635ef82 160000 --- a/vendor/nim-datastore +++ b/vendor/nim-datastore @@ -1 +1 @@ -Subproject commit 3ab6b84a634a7b2ee8c0144f050bf5893cd47c17 +Subproject commit b5635ef82f50255bc5ff2fe411722d870bcb587c diff --git a/vendor/nim-eth b/vendor/nim-eth index 15a09fab7..dcfbc4291 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 15a09fab737d08a2545284c727199c377bb0f4b7 +Subproject commit dcfbc4291d39b59563828c3e32be4d51a2f25931 diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 1ae2cd4a3..eea5811f0 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 1ae2cd4a35aa7c7ca21ca750fb7f951b3a6a97c0 +Subproject commit eea5811f00f7d72ed3b0edaa2f15332bf5382773 diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams index 720fc5e5c..cf8d4d226 160000 --- a/vendor/nim-faststreams +++ b/vendor/nim-faststreams @@ -1 +1 @@ -Subproject commit 720fc5e5c8e428d9d0af618e1e27c44b42350309 +Subproject commit cf8d4d22636b8e514caf17e49f9c786ac56b0e85 diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index be57dbc90..8bb1acbaa 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit be57dbc902d36f37540897e98c69aa80f868cb45 +Subproject commit 8bb1acbaa4b86eb866145b0d468eff64a57d1897 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 0408795be..274372132 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 0408795be95c00d75e96eaef6eae8a9c734014f5 +Subproject commit 274372132de497e6b7b793c9d5d5474b71bf80a2 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index 5127b26ee..6eadb6e93 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit 5127b26ee58076e9369e7c126c196793c2b12e73 +Subproject commit 6eadb6e939ffa7882ff5437033c11a9464d3385c diff --git a/vendor/nim-leopard b/vendor/nim-leopard index 895ff24ca..3e09d8113 160000 --- a/vendor/nim-leopard +++ b/vendor/nim-leopard @@ -1 +1 @@ -Subproject commit 895ff24ca6615d577acfb11811cdd5465f596c97 +Subproject commit 3e09d8113f874f3584c3fe93818541b2ff9fb9c3 diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index b29c22ba0..6da0cda88 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit b29c22ba0ef13de50b779c776830dbea1d50cd33 +Subproject commit 6da0cda88ab7780bd5fd342327adb91ab84692aa diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index b239791c5..036e110a6 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit b239791c568d9f9a76fd66d2322b2754700b6cc5 +Subproject commit 036e110a6080fba1a1662c58cfd8c21f9a548021 diff --git a/vendor/nim-metrics b/vendor/nim-metrics index 6142e433f..cacfdc124 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit 6142e433fc8ea9b73379770a788017ac528d46ff +Subproject commit cacfdc12454a0804c65112b9f4f50d1375208dcd diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 27d314d65..5e4059746 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 27d314d65c9078924b3239fe4e2f5af0c512b28c +Subproject commit 5e4059746e9095e1731b02eeaecd62a70fbe664d diff --git a/vendor/nim-nitro b/vendor/nim-nitro index 6b4c455bf..e3719433d 160000 --- a/vendor/nim-nitro +++ b/vendor/nim-nitro @@ -1 +1 @@ -Subproject commit 6b4c455bf4dad7449c1580055733a1738fcd5aec +Subproject commit e3719433d5ace25947c468787c805969642b3913 diff --git a/vendor/nim-poseidon2 b/vendor/nim-poseidon2 index 0346982f2..f5a351134 160000 --- a/vendor/nim-poseidon2 +++ b/vendor/nim-poseidon2 @@ -1 +1 @@ -Subproject commit 0346982f2c6891bcedd03d552af3a3bd57b2c1f9 +Subproject commit f5a3511349f77027e0b10cbdeb953c130cc8268c diff --git a/vendor/nim-presto b/vendor/nim-presto index c17bfdda2..92b1c7ff1 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit c17bfdda2c60cf5fadb043feb22e328b7659c719 +Subproject commit 92b1c7ff141e6920e1f8a98a14c35c1fa098e3be diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization index 28214b3e4..5a31137a8 160000 --- a/vendor/nim-protobuf-serialization +++ b/vendor/nim-protobuf-serialization @@ -1 +1 @@ -Subproject commit 28214b3e40c755a9886d2ec8f261ec48fbb6bec6 +Subproject commit 5a31137a82c2b6a989c9ed979bb636c7a49f570e diff --git a/vendor/nim-results b/vendor/nim-results index f3c666a27..df8113dda 160000 --- a/vendor/nim-results +++ b/vendor/nim-results @@ -1 +1 @@ -Subproject commit f3c666a272c69d70cb41e7245e7f6844797303ad +Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27 diff --git a/vendor/nim-serde b/vendor/nim-serde index 83e4a2ccf..69a7a0111 160000 --- a/vendor/nim-serde +++ b/vendor/nim-serde @@ -1 +1 @@ -Subproject commit 83e4a2ccf621d3040c6e7e0267393ca2d205988e +Subproject commit 69a7a0111addaa4aad885dd4bd7b5ee4684a06de diff --git a/vendor/nim-serialization b/vendor/nim-serialization index f709bd9e1..2086c9960 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit f709bd9e16b1b6870fe3e4401196479e014a2ef6 +Subproject commit 2086c99608b4bf472e1ef5fe063710f280243396 diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi index 362e1bd9f..05bbff1af 160000 --- a/vendor/nim-sqlite3-abi +++ b/vendor/nim-sqlite3-abi @@ -1 +1 @@ -Subproject commit 362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3 +Subproject commit 05bbff1af4e8fe2d972ba4b0667b89ca94d3ebba diff --git a/vendor/nim-stew b/vendor/nim-stew index 7afe7e3c0..a6e198132 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit 7afe7e3c070758cac1f628e4330109f3ef6fc853 +Subproject commit a6e198132097fb544d04959aeb3b839e1408f942 diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools index b3673c7a7..66585e2e9 160000 --- a/vendor/nim-taskpools +++ b/vendor/nim-taskpools @@ -1 +1 @@ -Subproject commit b3673c7a7a959ccacb393bd9b47e997bbd177f5a +Subproject commit 66585e2e960b7695e48ea60377fb3aeac96406e8 diff --git a/vendor/nim-testutils b/vendor/nim-testutils index b56a5953e..4d37244f9 160000 --- a/vendor/nim-testutils +++ b/vendor/nim-testutils @@ -1 +1 @@ -Subproject commit b56a5953e37fc5117bd6ea6dfa18418c5e112815 +Subproject commit 4d37244f9f5e1acd8592a4ceb5c3fc47bc160181 diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization index 86d477136..fea85b27f 160000 --- a/vendor/nim-toml-serialization +++ b/vendor/nim-toml-serialization @@ -1 +1 @@ -Subproject commit 86d477136f105f04bfd0dd7c0e939593d81fc581 +Subproject commit fea85b27f0badcf617033ca1bc05444b5fd8aa7a diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 index b178f4752..845b6af28 160000 --- a/vendor/nim-unittest2 +++ b/vendor/nim-unittest2 @@ -1 +1 @@ -Subproject commit b178f47527074964f76c395ad0dfc81cf118f379 +Subproject commit 845b6af28b9f68f02d320e03ad18eccccea7ddb9 diff --git a/vendor/nim-websock b/vendor/nim-websock index 2c3ae3137..ebe308a79 160000 --- a/vendor/nim-websock +++ b/vendor/nim-websock @@ -1 +1 @@ -Subproject commit 2c3ae3137f3c9cb48134285bd4a47186fa51f0e8 +Subproject commit ebe308a79a7b440a11dfbe74f352be86a3883508 diff --git a/vendor/nim-zlib b/vendor/nim-zlib index f34ca261e..91cf360b1 160000 --- a/vendor/nim-zlib +++ b/vendor/nim-zlib @@ -1 +1 @@ -Subproject commit f34ca261efd90f118dc1647beefd2f7a69b05d93 +Subproject commit 91cf360b1aeb2e0c753ff8bac6de22a41c5ed8cd diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index fe9bc3f37..8fafcd0ba 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit fe9bc3f3759ae1add6bf8c899db2e75327f03782 +Subproject commit 8fafcd0bac9f409091b7bcaee62ab6330f57441e diff --git a/vendor/nimcrypto b/vendor/nimcrypto index 24e006df8..dc07e3058 160000 --- a/vendor/nimcrypto +++ b/vendor/nimcrypto @@ -1 +1 @@ -Subproject commit 24e006df85927f64916e60511620583b11403178 +Subproject commit dc07e3058c6904eef965394493b6ea99aa2adefc diff --git a/vendor/npeg b/vendor/npeg index b15a10e38..409f6796d 160000 --- a/vendor/npeg +++ b/vendor/npeg @@ -1 +1 @@ -Subproject commit b15a10e388b91b898c581dbbcb6a718d46b27d2f +Subproject commit 409f6796d0e880b3f0222c964d1da7de6e450811 diff --git a/vendor/stint b/vendor/stint index 86621eced..5c5e01cef 160000 --- a/vendor/stint +++ b/vendor/stint @@ -1 +1 @@ -Subproject commit 86621eced1dcfb5e25903019ebcfc76ed9128ec5 +Subproject commit 5c5e01cef089a261474b7abfe246b37447aaa8ed diff --git a/vendor/upraises b/vendor/upraises index ff4f8108e..bc2628989 160000 --- a/vendor/upraises +++ b/vendor/upraises @@ -1 +1 @@ -Subproject commit ff4f8108e44fba9b35cac535ab63d3927e8fd3c2 +Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25 From 34f5e0071c0148798ecf499ad951d2ac2b53c992 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:28:40 +0100 Subject: [PATCH 04/44] Update Nim version to 2.0.12 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c502e23ac..64ed9c4c5 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ # # If NIM_COMMIT is set to "nimbusbuild", this will use the # version pinned by nimbus-build-system. -# PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21 -PINNED_NIM_VERSION := v2.0.6 +#PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21 +PINNED_NIM_VERSION := v2.0.12 ifeq ($(NIM_COMMIT),) NIM_COMMIT := $(PINNED_NIM_VERSION) else ifeq ($(NIM_COMMIT),pinned) From 7c9b607e38bc490faa249912ae3b247218b00cb5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:36:55 +0100 Subject: [PATCH 05/44] Add gcsafe pragma --- codex.nim | 2 +- codex/clock.nim | 2 +- codex/discovery.nim | 2 +- codex/erasure/backend.nim | 6 +++--- codex/stores/blockstore.nim | 26 +++++++++++++------------- codex/utils/timer.nim | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/codex.nim b/codex.nim index 79e0ef684..4ef012996 100644 --- a/codex.nim +++ b/codex.nim @@ -48,7 +48,7 @@ when isMainModule: let config = CodexConf.load( version = codexFullVersion, envVarsPrefix = "codex", - secondarySources = proc (config: CodexConf, sources: auto) = + secondarySources = proc (config: CodexConf, sources: auto) {.gcsafe, raises: [ConfigurationError].} = if configFile =? config.configFile: sources.addConfigFile(Toml, configFile) ) diff --git a/codex/clock.nim b/codex/clock.nim index f680ddec1..933cd199a 100644 --- a/codex/clock.nim +++ b/codex/clock.nim @@ -8,7 +8,7 @@ type SecondsSince1970* = int64 Timeout* = object of CatchableError -method now*(clock: Clock): SecondsSince1970 {.base, upraises: [].} = +method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, upraises: [].} = raiseAssert "not implemented" method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} = diff --git a/codex/discovery.nim b/codex/discovery.nim index 47ac950dd..277080f53 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -124,7 +124,7 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} = method removeProvider*( d: Discovery, - peerId: PeerId): Future[void] {.base.} = + peerId: PeerId): Future[void] {.base, gcsafe.} = ## Remove provider from providers table ## diff --git a/codex/erasure/backend.nim b/codex/erasure/backend.nim index 728c8bedb..873d800ed 100644 --- a/codex/erasure/backend.nim +++ b/codex/erasure/backend.nim @@ -22,7 +22,7 @@ type EncoderBackend* = ref object of ErasureBackend DecoderBackend* = ref object of ErasureBackend -method release*(self: ErasureBackend) {.base.} = +method release*(self: ErasureBackend) {.base, gcsafe.} = ## release the backend ## raiseAssert("not implemented!") @@ -31,7 +31,7 @@ method encode*( self: EncoderBackend, buffers, parity: var openArray[seq[byte]] -): Result[void, cstring] {.base.} = +): Result[void, cstring] {.base, gcsafe.} = ## encode buffers using a backend ## raiseAssert("not implemented!") @@ -41,7 +41,7 @@ method decode*( buffers, parity, recovered: var openArray[seq[byte]] -): Result[void, cstring] {.base.} = +): Result[void, cstring] {.base, gcsafe.} = ## decode buffers using a backend ## raiseAssert("not implemented!") diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index 52f375174..d3eafc067 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -33,13 +33,13 @@ type BlockStore* = ref object of RootObj onBlockStored*: ?CidCallback -method getBlock*(self: BlockStore, cid: Cid): Future[?!Block] {.base.} = +method getBlock*(self: BlockStore, cid: Cid): Future[?!Block] {.base, gcsafe.} = ## Get a block from the blockstore ## raiseAssert("getBlock by cid not implemented!") -method getBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Block] {.base.} = +method getBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Block] {.base, gcsafe.} = ## Get a block from the blockstore ## @@ -50,13 +50,13 @@ method getCid*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Cid] {. ## raiseAssert("getCid by treecid not implemented!") -method getBlock*(self: BlockStore, address: BlockAddress): Future[?!Block] {.base.} = +method getBlock*(self: BlockStore, address: BlockAddress): Future[?!Block] {.base, gcsafe.} = ## Get a block from the blockstore ## raiseAssert("getBlock by addr not implemented!") -method getBlockAndProof*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.base.} = +method getBlockAndProof*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.base, gcsafe.} = ## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree ## @@ -65,7 +65,7 @@ method getBlockAndProof*(self: BlockStore, treeCid: Cid, index: Natural): Future method putBlock*( self: BlockStore, blk: Block, - ttl = Duration.none): Future[?!void] {.base.} = + ttl = Duration.none): Future[?!void] {.base, gcsafe.} = ## Put a block to the blockstore ## @@ -76,7 +76,7 @@ method putCidAndProof*( treeCid: Cid, index: Natural, blockCid: Cid, - proof: CodexProof): Future[?!void] {.base.} = + proof: CodexProof): Future[?!void] {.base, gcsafe.} = ## Put a block proof to the blockstore ## @@ -85,7 +85,7 @@ method putCidAndProof*( method getCidAndProof*( self: BlockStore, treeCid: Cid, - index: Natural): Future[?!(Cid, CodexProof)] {.base.} = + index: Natural): Future[?!(Cid, CodexProof)] {.base, gcsafe.} = ## Get a block proof from the blockstore ## @@ -94,7 +94,7 @@ method getCidAndProof*( method ensureExpiry*( self: BlockStore, cid: Cid, - expiry: SecondsSince1970): Future[?!void] {.base.} = + expiry: SecondsSince1970): Future[?!void] {.base, gcsafe.} = ## Ensure that block's assosicated expiry is at least given timestamp ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact ## @@ -105,7 +105,7 @@ method ensureExpiry*( self: BlockStore, treeCid: Cid, index: Natural, - expiry: SecondsSince1970): Future[?!void] {.base.} = + expiry: SecondsSince1970): Future[?!void] {.base, gcsafe.} = ## Ensure that block's associated expiry is at least given timestamp ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact ## @@ -124,13 +124,13 @@ method delBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!void] raiseAssert("delBlock not implemented!") -method hasBlock*(self: BlockStore, cid: Cid): Future[?!bool] {.base.} = +method hasBlock*(self: BlockStore, cid: Cid): Future[?!bool] {.base, gcsafe.} = ## Check if the block exists in the blockstore ## raiseAssert("hasBlock not implemented!") -method hasBlock*(self: BlockStore, tree: Cid, index: Natural): Future[?!bool] {.base.} = +method hasBlock*(self: BlockStore, tree: Cid, index: Natural): Future[?!bool] {.base, gcsafe.} = ## Check if the block exists in the blockstore ## @@ -138,13 +138,13 @@ method hasBlock*(self: BlockStore, tree: Cid, index: Natural): Future[?!bool] {. method listBlocks*( self: BlockStore, - blockType = BlockType.Manifest): Future[?!AsyncIter[?Cid]] {.base.} = + blockType = BlockType.Manifest): Future[?!AsyncIter[?Cid]] {.base, gcsafe.} = ## Get the list of blocks in the BlockStore. This is an intensive operation ## raiseAssert("listBlocks not implemented!") -method close*(self: BlockStore): Future[void] {.base.} = +method close*(self: BlockStore): Future[void] {.base, gcsafe.} = ## Close the blockstore, cleaning up resources managed by it. ## For some implementations this may be a no-op ## diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index 9361d07b4..b74570385 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -40,7 +40,7 @@ proc timerLoop(timer: Timer) {.async.} = except CatchableError as exc: error "Timer caught unhandled exception: ", name=timer.name, msg=exc.msg -method start*(timer: Timer, callback: TimerCallback, interval: Duration) {.base.} = +method start*(timer: Timer, callback: TimerCallback, interval: Duration) {.gcsafe, base.} = if timer.loopFuture != nil: return trace "Timer starting: ", name=timer.name From 0e00f0c7cb806b40fcee4bf75a420f573178c1ce Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:41:08 +0100 Subject: [PATCH 06/44] Add missing import --- codex/discovery.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/codex/discovery.nim b/codex/discovery.nim index 277080f53..74ed0f6fd 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -17,6 +17,7 @@ import pkg/questionable/results import pkg/stew/shims/net import pkg/contractabi/address as ca import pkg/codexdht/discv5/[routing_table, protocol as discv5] +from pkg/nimcrypto import keccak256 import ./rng import ./errors From 0ecf3f7a78f67cbecefb13dc5cee8d655dbbe738 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:43:19 +0100 Subject: [PATCH 07/44] Update specific conf for Nim 2.x --- config.nims | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.nims b/config.nims index b64aacbdc..6a4767ad3 100644 --- a/config.nims +++ b/config.nims @@ -41,7 +41,7 @@ when defined(windows): # The dynamic Chronicles output currently prevents us from using colors on Windows # because these require direct manipulations of the stdout File object. - switch("define", "chronicles_colors=off") + switch("define", "chronicles_colors=NoColors") # This helps especially for 32-bit x86, which sans SSE2 and newer instructions # requires quite roundabout code generation for cryptography, and other 64-bit @@ -85,6 +85,8 @@ when (NimMajor, NimMinor) >= (1, 6): --warning:"DotLikeOps:off" when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11): --warning:"BareExcept:off" +when (NimMajor, NimMinor) >= (2, 0): + --mm:refc switch("define", "withoutPCRE") From 4935494c2982cfb189e970de68e32b15481227cd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:45:05 +0100 Subject: [PATCH 08/44] Fix method signatures --- codex/contracts/clock.nim | 2 +- codex/contracts/market.nim | 98 +++++++++++++++++++++++--------- codex/stores/repostore/store.nim | 19 ++++--- 3 files changed, 82 insertions(+), 37 deletions(-) diff --git a/codex/contracts/clock.nim b/codex/contracts/clock.nim index 937745bf4..1d03c1d64 100644 --- a/codex/contracts/clock.nim +++ b/codex/contracts/clock.nim @@ -45,7 +45,7 @@ method start*(clock: OnChainClock) {.async.} = if clock.started: return - proc onBlock(_: Block) = + proc onBlock(_: ?!Block) = # ignore block parameter; hardhat may call this with pending blocks asyncSpawn clock.update() diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index fdddcb220..a6a23b7c2 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -277,10 +277,14 @@ method canReserveSlot*( method subscribeRequests*(market: OnChainMarket, callback: OnRequest): Future[MarketSubscription] {.async.} = - proc onEvent(event: StorageRequested) {.upraises:[].} = - callback(event.requestId, - event.ask, - event.expiry) + proc onEvent(event: ?!StorageRequested) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId, + value.ask, + value.expiry) convertEthersError: let subscription = await market.contract.subscribe(StorageRequested, onEvent) @@ -289,8 +293,12 @@ method subscribeRequests*(market: OnChainMarket, method subscribeSlotFilled*(market: OnChainMarket, callback: OnSlotFilled): Future[MarketSubscription] {.async.} = - proc onEvent(event: SlotFilled) {.upraises:[].} = - callback(event.requestId, event.slotIndex) + proc onEvent(event: ?!SlotFilled) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId, value.slotIndex) convertEthersError: let subscription = await market.contract.subscribe(SlotFilled, onEvent) @@ -311,8 +319,12 @@ method subscribeSlotFilled*(market: OnChainMarket, method subscribeSlotFreed*(market: OnChainMarket, callback: OnSlotFreed): Future[MarketSubscription] {.async.} = - proc onEvent(event: SlotFreed) {.upraises:[].} = - callback(event.requestId, event.slotIndex) + proc onEvent(event: ?!SlotFreed) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId, value.slotIndex) convertEthersError: let subscription = await market.contract.subscribe(SlotFreed, onEvent) @@ -322,8 +334,12 @@ method subscribeSlotReservationsFull*( market: OnChainMarket, callback: OnSlotReservationsFull): Future[MarketSubscription] {.async.} = - proc onEvent(event: SlotReservationsFull) {.upraises:[].} = - callback(event.requestId, event.slotIndex) + proc onEvent(event: ?!SlotReservationsFull) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId, value.slotIndex) convertEthersError: let subscription = await market.contract.subscribe(SlotReservationsFull, onEvent) @@ -332,8 +348,12 @@ method subscribeSlotReservationsFull*( method subscribeFulfillment(market: OnChainMarket, callback: OnFulfillment): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestFulfilled) {.upraises:[].} = - callback(event.requestId) + proc onEvent(event: ?!RequestFulfilled) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestFulfilled, onEvent) @@ -343,9 +363,13 @@ method subscribeFulfillment(market: OnChainMarket, requestId: RequestId, callback: OnFulfillment): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestFulfilled) {.upraises:[].} = - if event.requestId == requestId: - callback(event.requestId) + proc onEvent(event: ?!RequestFulfilled) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + if value.requestId == requestId: + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestFulfilled, onEvent) @@ -354,8 +378,12 @@ method subscribeFulfillment(market: OnChainMarket, method subscribeRequestCancelled*(market: OnChainMarket, callback: OnRequestCancelled): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestCancelled) {.upraises:[].} = - callback(event.requestId) + proc onEvent(event: ?!RequestCancelled) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestCancelled, onEvent) @@ -365,9 +393,13 @@ method subscribeRequestCancelled*(market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestCancelled) {.upraises:[].} = - if event.requestId == requestId: - callback(event.requestId) + proc onEvent(event: ?!RequestCancelled) {.upraises:[].} = + without value =? event: + error "The event object is not defined" + return + + if value.requestId == requestId: + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestCancelled, onEvent) @@ -376,8 +408,12 @@ method subscribeRequestCancelled*(market: OnChainMarket, method subscribeRequestFailed*(market: OnChainMarket, callback: OnRequestFailed): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestFailed) {.upraises:[]} = - callback(event.requestId) + proc onEvent(event: ?!RequestFailed) {.upraises:[]} = + without value =? event: + error "The event object is not defined" + return + + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestFailed, onEvent) @@ -387,9 +423,13 @@ method subscribeRequestFailed*(market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed): Future[MarketSubscription] {.async.} = - proc onEvent(event: RequestFailed) {.upraises:[]} = - if event.requestId == requestId: - callback(event.requestId) + proc onEvent(event: ?!RequestFailed) {.upraises:[]} = + without value =? event: + error "The event object is not defined" + return + + if value.requestId == requestId: + callback(value.requestId) convertEthersError: let subscription = await market.contract.subscribe(RequestFailed, onEvent) @@ -398,8 +438,12 @@ method subscribeRequestFailed*(market: OnChainMarket, method subscribeProofSubmission*(market: OnChainMarket, callback: OnProofSubmitted): Future[MarketSubscription] {.async.} = - proc onEvent(event: ProofSubmitted) {.upraises: [].} = - callback(event.id) + proc onEvent(event: ?!ProofSubmitted) {.upraises: [].} = + without value =? event: + error "The event object is not defined" + return + + callback(value.id) convertEthersError: let subscription = await market.contract.subscribe(ProofSubmitted, onEvent) diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index 5ff99e649..63c59d2b6 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -323,15 +323,16 @@ method getBlockExpirations*( return failure(err) let - filteredIter = await asyncQueryIter.filterSuccess() - blockExpIter = await mapFilter[KeyVal[BlockMetadata], BlockExpiration](filteredIter, - proc (kv: KeyVal[BlockMetadata]): Future[?BlockExpiration] {.async.} = - without cid =? Cid.init(kv.key.value).mapFailure, err: - error "Failed decoding cid", err = err.msg - return BlockExpiration.none - - BlockExpiration(cid: cid, expiry: kv.value.expiry).some - ) + filteredIter: AsyncIter[KeyVal[BlockMetadata]] = await asyncQueryIter.filterSuccess() + + proc mapping (kv: KeyVal[BlockMetadata]): Future[?BlockExpiration] {.async.} = + without cid =? Cid.init(kv.key.value).mapFailure, err: + error "Failed decoding cid", err = err.msg + return BlockExpiration.none + + BlockExpiration(cid: cid, expiry: kv.value.expiry).some + + let blockExpIter = await mapFilter[KeyVal[BlockMetadata], BlockExpiration](filteredIter, mapping) success(blockExpIter) From a1c8c94ada2fa993ef6dcd2a88d1988513415a41 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:46:45 +0100 Subject: [PATCH 09/44] Revert erasure coding attempt to fix bug --- codex/codex.nim | 16 ++--------- codex/erasure/erasure.nim | 46 ++++++++++++++++++------------ codex/node.nim | 17 ++++------- tests/codex/node/helpers.nim | 7 +---- tests/codex/node/testcontracts.nim | 4 +-- tests/codex/node/testnode.nim | 4 +-- tests/codex/testerasure.nim | 6 +--- 7 files changed, 39 insertions(+), 61 deletions(-) diff --git a/codex/codex.nim b/codex/codex.nim index 441bdf883..ffdf3b41b 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -11,7 +11,6 @@ import std/sequtils import std/strutils import std/os import std/tables -import std/cpuinfo import pkg/chronos import pkg/presto @@ -24,7 +23,6 @@ import pkg/stew/shims/net as stewnet import pkg/datastore import pkg/ethers except Rng import pkg/stew/io2 -import pkg/taskpools import ./node import ./conf @@ -55,7 +53,6 @@ type codexNode: CodexNodeRef repoStore: RepoStore maintenance: BlockMaintainer - taskpool: Taskpool CodexPrivateKey* = libp2p.PrivateKey # alias EthWallet = ethers.Wallet @@ -190,10 +187,6 @@ proc start*(s: CodexServer) {.async.} = proc stop*(s: CodexServer) {.async.} = notice "Stopping codex node" - - s.taskpool.syncAll() - s.taskpool.shutdown() - await allFuturesThrowing( s.restServer.stop(), s.codexNode.switch.stop(), @@ -283,15 +276,12 @@ proc new*( else: none Prover - taskpool = Taskpool.new(num_threads = countProcessors()) - codexNode = CodexNodeRef.new( switch = switch, networkStore = store, engine = engine, - prover = prover, discovery = discovery, - taskpool = taskpool) + prover = prover) restServer = RestServerRef.new( codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), @@ -306,6 +296,4 @@ proc new*( config: config, codexNode: codexNode, restServer: restServer, - repoStore: repoStore, - maintenance: maintenance, - taskpool: taskpool) + repoStore: repoStore) diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 56e3e1cf5..d35fc18de 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -17,7 +17,6 @@ import std/sugar import pkg/chronos import pkg/libp2p/[multicodec, cid, multihash] import pkg/libp2p/protobuf/minprotobuf -import pkg/taskpools import ../logutils import ../manifest @@ -32,7 +31,6 @@ import ../errors import pkg/stew/byteutils import ./backend -import ./asyncbackend export backend @@ -73,7 +71,6 @@ type encoderProvider*: EncoderProvider decoderProvider*: DecoderProvider store*: BlockStore - taskpool: Taskpool EncodingParams = object ecK: Natural @@ -295,23 +292,30 @@ proc encodeData( # TODO: Don't allocate a new seq every time, allocate once and zero out var data = seq[seq[byte]].new() # number of blocks to encode + parityData = newSeqWith[seq[byte]](params.ecM, newSeq[byte](manifest.blockSize.int)) data[].setLen(params.ecK) + # TODO: this is a tight blocking loop so we sleep here to allow + # other events to be processed, this should be addressed + # by threading + await sleepAsync(10.millis) without resolved =? (await self.prepareEncodingData(manifest, params, step, data, cids, emptyBlock)), err: trace "Unable to prepare data", error = err.msg return failure(err) - trace "Erasure coding data", data = data[].len, parity = params.ecM + trace "Erasure coding data", data = data[].len, parity = parityData.len - without parity =? await asyncEncode(self.taskpool, encoder, data, manifest.blockSize.int, params.ecM), err: - trace "Error encoding data", err = err.msg - return failure(err) + if ( + let res = encoder.encode(data[], parityData); + res.isErr): + trace "Unable to encode manifest!", error = $res.error + return failure($res.error) var idx = params.rounded + step for j in 0.. Date: Wed, 18 Dec 2024 16:47:54 +0100 Subject: [PATCH 10/44] More gcsafe pragma --- codex/utils/asyncstatemachine.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 3f15af319..ccbff0639 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -25,7 +25,7 @@ logScope: proc new*[T: Machine](_: type T): T = T(trackedFutures: TrackedFutures.new()) -method `$`*(state: State): string {.base.} = +method `$`*(state: State): string {.base, gcsafe.} = raiseAssert "not implemented" proc transition(_: type Event, previous, next: State): Event = @@ -62,7 +62,7 @@ proc run(machine: Machine, state: State) {.async.} = if next =? await state.run(machine): machine.schedule(Event.transition(state, next)) -proc scheduler(machine: Machine) {.async.} = +proc scheduler(machine: Machine) {.async, gcsafe.} = var running: Future[void] while machine.started: let event = await machine.scheduled.get().track(machine) @@ -74,13 +74,15 @@ proc scheduler(machine: Machine) {.async.} = machine.state = next debug "enter state", state = fromState & " => " & $machine.state running = machine.run(machine.state) + + proc catchError(err: ref CatchableError) {.gcsafe.} = + trace "error caught in state.run, calling state.onError", state = $machine.state + machine.schedule(machine.onError(err)) + running .track(machine) .cancelled(proc() = trace "state.run cancelled, swallowing", state = $machine.state) - .catch(proc(err: ref CatchableError) = - trace "error caught in state.run, calling state.onError", state = $machine.state - machine.schedule(machine.onError(err)) - ) + .catch(catchError) proc start*(machine: Machine, initialState: State) = if machine.started: From a2f0528dc4b077ca3e14b646aa6882f6d0c8be6e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:48:28 +0100 Subject: [PATCH 11/44] Duplicate code from libp2p because it is not exported anymore --- codex/errors.nim | 23 +++++++++++++++++++++++ codex/merkletree/codex/codex.nim | 8 ++++++++ 2 files changed, 31 insertions(+) diff --git a/codex/errors.nim b/codex/errors.nim index 9947d4b72..d98bfc041 100644 --- a/codex/errors.nim +++ b/codex/errors.nim @@ -38,6 +38,29 @@ func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} = else: T.failure("Option is None") +# allFuturesThrowing was moved to the tests in libp2p +proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] = + var futs: seq[Future[T]] + for fut in args: + futs &= fut + proc call() {.async.} = + var first: ref CatchableError = nil + futs = await allFinished(futs) + for fut in futs: + if fut.failed: + let err = fut.readError() + if err of Defect: + raise err + else: + if err of CancelledError: + raise err + if isNil(first): + first = err + if not isNil(first): + raise first + + return call() + proc allFutureResult*[T](fut: seq[Future[T]]): Future[?!void] {.async.} = try: await allFuturesThrowing(fut) diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 974123f2f..72b044f2d 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -47,6 +47,14 @@ type CodexProof* = ref object of ByteProof mcodec*: MultiCodec +# CodeHashes is not exported from libp2p +# So we need to recreate it instead of +proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} = + for item in HashesList: + result[item.mcodec] = item + +const CodeHashes = initMultiHashCodeTable() + func mhash*(mcodec: MultiCodec): ?!MHash = let mhash = CodeHashes.getOrDefault(mcodec) From a2f09b22969be6acf99cbc5804318ce0a6306ab9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:49:19 +0100 Subject: [PATCH 12/44] Fix camelcase function names --- codex/merkletree/poseidon2.nim | 8 ++--- codex/slots/proofs/backends/circomcompat.nim | 38 ++++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/codex/merkletree/poseidon2.nim b/codex/merkletree/poseidon2.nim index 2e56a4b96..eec8d6115 100644 --- a/codex/merkletree/poseidon2.nim +++ b/codex/merkletree/poseidon2.nim @@ -24,10 +24,10 @@ import ./merkletree export merkletree, poseidon2 const - KeyNoneF = F.fromhex("0x0") - KeyBottomLayerF = F.fromhex("0x1") - KeyOddF = F.fromhex("0x2") - KeyOddAndBottomLayerF = F.fromhex("0x3") + KeyNoneF = F.fromHex("0x0") + KeyBottomLayerF = F.fromHex("0x1") + KeyOddF = F.fromHex("0x2") + KeyOddAndBottomLayerF = F.fromHex("0x3") Poseidon2Zero* = zero diff --git a/codex/slots/proofs/backends/circomcompat.nim b/codex/slots/proofs/backends/circomcompat.nim index 8619457a4..374b8151c 100644 --- a/codex/slots/proofs/backends/circomcompat.nim +++ b/codex/slots/proofs/backends/circomcompat.nim @@ -76,7 +76,7 @@ proc release*(self: CircomCompat) = ## if not isNil(self.backendCfg): - self.backendCfg.unsafeAddr.releaseCfg() + self.backendCfg.unsafeAddr.release_cfg() if not isNil(self.vkp): self.vkp.unsafeAddr.release_key() @@ -102,9 +102,9 @@ proc prove[H]( defer: if ctx != nil: - ctx.addr.releaseCircomCompat() + ctx.addr.release_circom_compat() - if initCircomCompat( + if init_circom_compat( self.backendCfg, addr ctx) != ERR_OK or ctx == nil: raiseAssert("failed to initialize CircomCompat ctx") @@ -114,27 +114,27 @@ proc prove[H]( dataSetRoot = input.datasetRoot.toBytes slotRoot = input.slotRoot.toBytes - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "entropy".cstring, entropy[0].addr, entropy.len.uint32) != ERR_OK: return failure("Failed to push entropy") - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "dataSetRoot".cstring, dataSetRoot[0].addr, dataSetRoot.len.uint32) != ERR_OK: return failure("Failed to push data set root") - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK: return failure("Failed to push data set root") - if ctx.pushInputU32( + if ctx.push_input_u32( "nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK: return failure("Failed to push nCellsPerSlot") - if ctx.pushInputU32( + if ctx.push_input_u32( "nSlotsPerDataSet".cstring, input.nSlotsPerDataSet.uint32) != ERR_OK: return failure("Failed to push nSlotsPerDataSet") - if ctx.pushInputU32( + if ctx.push_input_u32( "slotIndex".cstring, input.slotIndex.uint32) != ERR_OK: return failure("Failed to push slotIndex") @@ -143,7 +143,7 @@ proc prove[H]( doAssert(slotProof.len == self.datasetDepth) # arrays are always flattened - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "slotProof".cstring, slotProof[0].addr, uint (slotProof[0].len * slotProof.len)) != ERR_OK: @@ -154,13 +154,13 @@ proc prove[H]( merklePaths = s.merklePaths.mapIt( it.toBytes ) data = s.cellData.mapIt( @(it.toBytes) ).concat - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "merklePaths".cstring, merklePaths[0].addr, uint (merklePaths[0].len * merklePaths.len)) != ERR_OK: return failure("Failed to push merkle paths") - if ctx.pushInputU256Array( + if ctx.push_input_u256_array( "cellData".cstring, data[0].addr, data.len.uint) != ERR_OK: @@ -172,7 +172,7 @@ proc prove[H]( let proof = try: if ( - let res = self.backendCfg.proveCircuit(ctx, proofPtr.addr); + let res = self.backendCfg.prove_circuit(ctx, proofPtr.addr); res != ERR_OK) or proofPtr == nil: return failure("Failed to prove - err code: " & $res) @@ -180,7 +180,7 @@ proc prove[H]( proofPtr[] finally: if proofPtr != nil: - proofPtr.addr.releaseProof() + proofPtr.addr.release_proof() success proof @@ -202,7 +202,7 @@ proc verify*[H]( inputs = inputs.toCircomInputs() try: - let res = verifyCircuit(proofPtr, inputs.addr, self.vkp) + let res = verify_circuit(proofPtr, inputs.addr, self.vkp) if res == ERR_OK: success true elif res == ERR_FAILED_TO_VERIFY_PROOF: @@ -228,18 +228,18 @@ proc init*( var cfg: ptr CircomBn254Cfg var zkey = if zkeyPath.len > 0: zkeyPath.cstring else: nil - if initCircomConfig( + if init_circom_config( r1csPath.cstring, wasmPath.cstring, zkey, cfg.addr) != ERR_OK or cfg == nil: - if cfg != nil: cfg.addr.releaseCfg() + if cfg != nil: cfg.addr.release_cfg() raiseAssert("failed to initialize circom compat config") var vkpPtr: ptr VerifyingKey = nil - if cfg.getVerifyingKey(vkpPtr.addr) != ERR_OK or vkpPtr == nil: - if vkpPtr != nil: vkpPtr.addr.releaseKey() + if cfg.get_verifying_key(vkpPtr.addr) != ERR_OK or vkpPtr == nil: + if vkpPtr != nil: vkpPtr.addr.release_key() raiseAssert("Failed to get verifying key") CircomCompat( From 902c6dc692f6fe4ce53811622218f01eb482d801 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:50:00 +0100 Subject: [PATCH 13/44] Use alreadySeen because need is not a bool anymore --- codex/rest/json.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 24936b34e..4a0009430 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -90,7 +90,7 @@ proc init*(_: type RestNode, node: dn.Node): RestNode = peerId: node.record.data.peerId, record: node.record, address: node.address, - seen: node.seen + seen: node.alreadySeen() ) proc init*(_: type RestRoutingTable, routingTable: rt.RoutingTable): RestRoutingTable = From 83b4943c177a80e37da76dee60bbca67404d7e16 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 16:51:07 +0100 Subject: [PATCH 14/44] newLPStreamReadError does not exist anymore so use another error --- codex/streams/storestream.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 8a3b1a3cf..009a1a8a8 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -102,7 +102,7 @@ method readOnce*( # Read contents of block `blockNum` without blk =? await self.store.getBlock(address), error: - raise newLPStreamReadError(error) + raise newLPStreamConnDownError(error) trace "Reading bytes from store stream", manifestCid = self.manifest.cid.get(), numBlocks = self.manifest.blocksCount, blockNum, blkCid = blk.cid, bytes = readBytes, blockOffset From daadffe30a10d9509f98d807d385592297afa13d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 18 Dec 2024 17:17:31 +0100 Subject: [PATCH 15/44] Replace ValidIpAddress by IpAddress --- codex.nim | 4 ++-- codex/conf.nim | 40 ++++++++++++++++++++++++++------------- codex/discovery.nim | 6 +++--- codex/utils/addrutils.nim | 2 +- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/codex.nim b/codex.nim index 4ef012996..5faf33a07 100644 --- a/codex.nim +++ b/codex.nim @@ -55,11 +55,11 @@ when isMainModule: config.setupLogging() config.setupMetrics() - if config.nat == ValidIpAddress.init(IPv4_any()): + if config.nat == IPv4_any(): error "`--nat` cannot be set to the any (`0.0.0.0`) address" quit QuitFailure - if config.nat == ValidIpAddress.init("127.0.0.1"): + if config.nat == static parseIpAddress("127.0.0.1"): warn "`--nat` is set to loopback, your node wont properly announce over the DHT" if not(checkAndCreateDataDir((config.dataDir).string)): diff --git a/codex/conf.nim b/codex/conf.nim index d59b347e8..d4a4ec5da 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -114,9 +114,9 @@ type metricsAddress* {. desc: "Listening address of the metrics server" - defaultValue: ValidIpAddress.init("127.0.0.1") + defaultValue: defaultAddress(config) defaultValueDesc: "127.0.0.1" - name: "metrics-address" }: ValidIpAddress + name: "metrics-address" }: IpAddress metricsPort* {. desc: "Listening HTTP port of the metrics server" @@ -142,17 +142,17 @@ type # TODO: change this once we integrate nat support nat* {. desc: "IP Addresses to announce behind a NAT" - defaultValue: ValidIpAddress.init("127.0.0.1") + defaultValue: defaultAddress(config) defaultValueDesc: "127.0.0.1" abbr: "a" - name: "nat" }: ValidIpAddress + name: "nat" }: IpAddress discoveryIp* {. desc: "Discovery listen address" - defaultValue: ValidIpAddress.init(IPv4_any()) + defaultValue: IPv4_any() defaultValueDesc: "0.0.0.0" abbr: "e" - name: "disc-ip" }: ValidIpAddress + name: "disc-ip" }: IpAddress discoveryPort* {. desc: "Discovery (UDP) port" @@ -413,6 +413,9 @@ type logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog logutils.formatIt(LogFormat.json, EthAddress): %it +func defaultAddress*(conf: CodexConf): IpAddress = + result = static parseIpAddress("127.0.0.1") + func persistence*(self: CodexConf): bool = self.cmd == StartUpCmd.persistence @@ -445,13 +448,17 @@ const proc parseCmdArg*(T: typedesc[MultiAddress], input: string): MultiAddress - {.upraises: [ValueError, LPError].} = + {.upraises: [ValueError] .} = var ma: MultiAddress - let res = MultiAddress.init(input) - if res.isOk: - ma = res.get() - else: - warn "Invalid MultiAddress", input=input, error = res.error() + try: + let res = MultiAddress.init(input) + if res.isOk: + ma = res.get() + else: + warn "Invalid MultiAddress", input=input, error = res.error() + quit QuitFailure + except LPError as exc: + warn "Invalid MultiAddress uri", uri = input, error = exc.msg quit QuitFailure ma @@ -461,6 +468,9 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = if not res.fromURI(uri): warn "Invalid SignedPeerRecord uri", uri = uri quit QuitFailure + except LPError as exc: + warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg + quit QuitFailure except CatchableError as exc: warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg quit QuitFailure @@ -494,7 +504,11 @@ proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = error "invalid SignedPeerRecord configuration value", error = err.msg quit QuitFailure - val = SignedPeerRecord.parseCmdArg(uri) + try: + val = SignedPeerRecord.parseCmdArg(uri) + except LPError as err: + warn "Invalid SignedPeerRecord uri", uri = uri, error = err.msg + quit QuitFailure proc readValue*(r: var TomlReader, val: var MultiAddress) = without input =? r.readValue(string).catch, err: diff --git a/codex/discovery.nim b/codex/discovery.nim index 74ed0f6fd..d79aee756 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -147,7 +147,7 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = d.protocol.updateRecord(d.providerRecord) .expect("Should update SPR") -proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) = +proc updateDhtRecord*(d: Discovery, ip: IpAddress, port: Port) = ## Update providers record ## @@ -173,7 +173,7 @@ proc stop*(d: Discovery) {.async.} = proc new*( T: type Discovery, key: PrivateKey, - bindIp = ValidIpAddress.init(IPv4_any()), + bindIp = IPv4_any(), bindPort = 0.Port, announceAddrs: openArray[MultiAddress], bootstrapNodes: openArray[SignedPeerRecord] = [], @@ -203,7 +203,7 @@ proc new*( self.protocol = newProtocol( key, - bindIp = bindIp.toNormalIp, + bindIp = bindIp, bindPort = bindPort, record = self.providerRecord.get, bootstrapRecords = bootstrapNodes, diff --git a/codex/utils/addrutils.nim b/codex/utils/addrutils.nim index f044581a0..0b2c56f0e 100644 --- a/codex/utils/addrutils.nim +++ b/codex/utils/addrutils.nim @@ -18,7 +18,7 @@ import pkg/stew/shims/net func remapAddr*( address: MultiAddress, - ip: Option[ValidIpAddress] = ValidIpAddress.none, + ip: Option[IpAddress] = IpAddress.none, port: Option[Port] = Port.none ): MultiAddress = ## Remap addresses to new IP and/or Port From d6b1781b34b28c256ba32e1cb0a5e4bfff4b4c67 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:14:28 +0100 Subject: [PATCH 16/44] Add gcsafe pragma --- codex/slots/proofs/backendfactory.nim | 2 +- codex/slots/proofs/backendutils.nim | 2 +- codex/stores/blockstore.nim | 4 ++-- tests/integration/nodeprocess.nim | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/codex/slots/proofs/backendfactory.nim b/codex/slots/proofs/backendfactory.nim index 80dc1b8e5..ac478e1ad 100644 --- a/codex/slots/proofs/backendfactory.nim +++ b/codex/slots/proofs/backendfactory.nim @@ -43,7 +43,7 @@ proc zkeyFilePath(config: CodexConf): string = proc initializeFromCircuitDirFiles( config: CodexConf, - utils: BackendUtils): ?!AnyBackend = + utils: BackendUtils): ?!AnyBackend {.gcsafe.} = if fileExists(config.r1csFilePath) and fileExists(config.wasmFilePath) and fileExists(config.zkeyFilePath): diff --git a/codex/slots/proofs/backendutils.nim b/codex/slots/proofs/backendutils.nim index 19740acb7..f7e6e2e14 100644 --- a/codex/slots/proofs/backendutils.nim +++ b/codex/slots/proofs/backendutils.nim @@ -8,5 +8,5 @@ method initializeCircomBackend*( r1csFile: string, wasmFile: string, zKeyFile: string -): AnyBackend {.base.} = +): AnyBackend {.base, gcsafe.}= CircomCompat.init(r1csFile, wasmFile, zKeyFile) diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index d3eafc067..791e7d5b1 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -112,13 +112,13 @@ method ensureExpiry*( raiseAssert("Not implemented!") -method delBlock*(self: BlockStore, cid: Cid): Future[?!void] {.base.} = +method delBlock*(self: BlockStore, cid: Cid): Future[?!void] {.base, gcsafe.} = ## Delete a block from the blockstore ## raiseAssert("delBlock not implemented!") -method delBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!void] {.base.} = +method delBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!void] {.base, gcsafe.} = ## Delete a block from the blockstore ## diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index 61947c20c..c21fb7578 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -26,22 +26,22 @@ type name*: string NodeProcessError* = object of CatchableError -method workingDir(node: NodeProcess): string {.base.} = +method workingDir(node: NodeProcess): string {.base, gcsafe.} = raiseAssert "not implemented" -method executable(node: NodeProcess): string {.base.} = +method executable(node: NodeProcess): string {.base, gcsafe.} = raiseAssert "not implemented" -method startedOutput(node: NodeProcess): string {.base.} = +method startedOutput(node: NodeProcess): string {.base, gcsafe.} = raiseAssert "not implemented" -method processOptions(node: NodeProcess): set[AsyncProcessOption] {.base.} = +method processOptions(node: NodeProcess): set[AsyncProcessOption] {.base, gcsafe.} = raiseAssert "not implemented" -method outputLineEndings(node: NodeProcess): string {.base.} = +method outputLineEndings(node: NodeProcess): string {.base, gcsafe.} = raiseAssert "not implemented" -method onOutputLineCaptured(node: NodeProcess, line: string) {.base.} = +method onOutputLineCaptured(node: NodeProcess, line: string) {.base, gcsafe.} = raiseAssert "not implemented" method start*(node: NodeProcess) {.base, async.} = From 0724d7f8f10beb2f101852c8b03941ff8cde85c4 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:15:18 +0100 Subject: [PATCH 17/44] Restore maintenance parameter deleted by mistake when removing esasure coding fix attempt code --- codex/codex.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codex/codex.nim b/codex/codex.nim index ffdf3b41b..8c204b559 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -296,4 +296,5 @@ proc new*( config: config, codexNode: codexNode, restServer: restServer, - repoStore: repoStore) + repoStore: repoStore, + maintenance: maintenance) From 6ad496dcbade185f8b8924ecee230b0acd191581 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:18:00 +0100 Subject: [PATCH 18/44] Update method signatures --- codex/streams/asyncstreamwrapper.nim | 6 +++--- tests/codex/testchunking.nim | 29 +++++++++++++++++++-------- tests/integration/testecbug.nim | 8 ++++++-- tests/integration/testmarketplace.nim | 9 +++++++-- tests/integration/testproofs.nim | 26 +++++++++++++++++------- 5 files changed, 56 insertions(+), 22 deletions(-) diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index a8a559557..327643b40 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -64,7 +64,7 @@ method readOnce*( self: AsyncStreamWrapper, pbytes: pointer, nbytes: int -): Future[int] {.async.} = +): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} = trace "Reading bytes from reader", bytes = nbytes if isNil(self.reader): @@ -118,7 +118,7 @@ method closed*(self: AsyncStreamWrapper): bool = method atEof*(self: AsyncStreamWrapper): bool = self.reader.atEof() -method closeImpl*(self: AsyncStreamWrapper) {.async.} = +method closeImpl*(self: AsyncStreamWrapper) {.async: (raises: []).} = try: trace "Shutting down async chronos stream" if not self.closed(): @@ -130,7 +130,7 @@ method closeImpl*(self: AsyncStreamWrapper) {.async.} = trace "Shutdown async chronos stream" except CancelledError as exc: - raise exc + error "Error received cancelled error when closing chronos stream", msg = exc.msg except CatchableError as exc: trace "Error closing async chronos stream", msg = exc.msg diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 3a4c2301c..3a8c0b4b8 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -6,16 +6,23 @@ import pkg/chronos import ../asynctest import ./helpers +# Trying to use a CancelledError or LPStreamError value for toRaise +# will produce a compilation error; +# Error: only a 'ref object' can be raised +# This is because they are not ref object but plain object. +# CancelledError* = object of FutureError +# LPStreamError* = object of LPError + type CrashingStreamWrapper* = ref object of LPStream - toRaise*: ref CatchableError + toRaise*: proc(): void {.gcsafe, raises: [CancelledError, LPStreamError].} method readOnce*( self: CrashingStreamWrapper, pbytes: pointer, nbytes: int -): Future[int] {.async.} = - raise self.toRaise +): Future[int] {.gcsafe, async: (raises: [CancelledError, LPStreamError]).} = + self.toRaise() asyncchecksuite "Chunking": test "should return proper size chunks": @@ -88,13 +95,14 @@ asyncchecksuite "Chunking": string.fromBytes(data) == readFile(path) fileChunker.offset == data.len - proc raiseStreamException(exc: ref CatchableError) {.async.} = + proc raiseStreamException(exc: ref CancelledError | ref LPStreamError) {.async.} = let stream = CrashingStreamWrapper.new() let chunker = LPStreamChunker.new( stream = stream, chunkSize = 2'nb) - stream.toRaise = exc + stream.toRaise = proc(): void {.raises: [CancelledError, LPStreamError].} = + raise exc discard (await chunker.getBytes()) test "stream should forward LPStreamError": @@ -112,6 +120,11 @@ asyncchecksuite "Chunking": expect LPStreamError: await raiseStreamException(newException(LPStreamError, "test error")) - test "stream should convert other exceptions to defect": - expect Defect: - await raiseStreamException(newException(CatchableError, "test error")) + # This test cannot exist anymore. + # The signature of the method readOnce is explicitly listing the error raised: + # LPStreamError and CancelledError. + # So trying to raise CatchableError will not compile because CatchableError is + # not in the pragma array amd Nim 2 looks stricter for this rule. + # test "stream should convert other exceptions to defect": + # expect Defect: + # await raiseStreamException(newException(CatchableError, "test error")) diff --git a/tests/integration/testecbug.nim b/tests/integration/testecbug.nim index d33c857a1..ca00ec4a2 100644 --- a/tests/integration/testecbug.nim +++ b/tests/integration/testecbug.nim @@ -33,8 +33,12 @@ marketplacesuite "Bug #821 - node crashes during erasure coding": let cid = clientApi.upload(data).get var requestId = none RequestId - proc onStorageRequested(event: StorageRequested) {.raises:[].} = - requestId = event.requestId.some + proc onStorageRequested(event: ?!StorageRequested) {.raises:[].} = + without value =? event: + trace "The onSlotFilled event is not defined." + discard + + requestId = value.requestId.some let subscription = await marketplace.subscribe(StorageRequested, onStorageRequested) diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index 6e62c58b3..888235cbb 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -135,8 +135,13 @@ marketplacesuite "Marketplace payouts": let cid = clientApi.upload(data).get var slotIdxFilled = none UInt256 - proc onSlotFilled(event: SlotFilled) = - slotIdxFilled = some event.slotIndex + + let onSlotFilled = proc (event: ?!SlotFilled) = + without value =? event: + trace "The onSlotFilled event is not defined." + discard + + slotIdxFilled = value.slotIndex.some let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 0d8b9bd3e..3b6ab41fe 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -54,7 +54,7 @@ marketplacesuite "Hosts submit regular proofs": check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) var proofWasSubmitted = false - proc onProofSubmitted(event: ProofSubmitted) = + proc onProofSubmitted(event: ?!ProofSubmitted) = proofWasSubmitted = true let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) @@ -120,8 +120,12 @@ marketplacesuite "Simulate invalid proofs": check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: + proc onSlotFreed(event: ?!SlotFreed) = + without value =? event: + trace "The onSlotFreed event is not defined." + discard + + if value.requestId == requestId: slotWasFreed = true let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) @@ -176,8 +180,12 @@ marketplacesuite "Simulate invalid proofs": let requestId = client0.requestId(purchaseId).get var slotWasFilled = false - proc onSlotFilled(event: SlotFilled) = - if event.requestId == requestId: + proc onSlotFilled(event: ?!SlotFilled) = + without value =? event: + trace "The onSlotFilled event is not defined." + discard + + if value.requestId == requestId: slotWasFilled = true let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) @@ -185,8 +193,12 @@ marketplacesuite "Simulate invalid proofs": check eventually(slotWasFilled, timeout = expiry.int * 1000) var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: + proc onSlotFreed(event: ?!SlotFreed) = + without value =? event: + trace "The onSlotFreed event is not defined." + discard + + if value.requestId == requestId: slotWasFreed = true let freedSubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) From 5c80a84982e246dce401c9b53fb2a172f679707c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:18:34 +0100 Subject: [PATCH 19/44] Copy LPStreamReadError code from libp2p which was removed --- codex/streams/storestream.nim | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 009a1a8a8..bdfcc4392 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -73,11 +73,20 @@ proc `size=`*(self: StoreStream, size: int) method atEof*(self: StoreStream): bool = self.offset >= self.size +type LPStreamReadError* = object of LPStreamError + par*: ref CatchableError + +proc newLPStreamReadError*(p: ref CatchableError): ref LPStreamReadError = + var w = newException(LPStreamReadError, "Read stream failed") + w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg + w.par = p + result = w + method readOnce*( self: StoreStream, pbytes: pointer, nbytes: int -): Future[int] {.async.} = +): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} = ## Read `nbytes` from current position in the StoreStream into output buffer pointed by `pbytes`. ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. @@ -100,9 +109,10 @@ method readOnce*( self.manifest.blockSize.int - blockOffset]) address = BlockAddress(leaf: true, treeCid: self.manifest.treeCid, index: blockNum) + # Read contents of block `blockNum` - without blk =? await self.store.getBlock(address), error: - raise newLPStreamConnDownError(error) + without blk =? (await self.store.getBlock(address)).tryGet.catch, error: + raise newLPStreamReadError(error) trace "Reading bytes from store stream", manifestCid = self.manifest.cid.get(), numBlocks = self.manifest.blocksCount, blockNum, blkCid = blk.cid, bytes = readBytes, blockOffset From 6b64c253f1d3985ceefde26342f56eecea29bb9e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:19:06 +0100 Subject: [PATCH 20/44] Fix camel case --- tests/codex/merkletree/helpers.nim | 2 +- tests/codex/node/helpers.nim | 4 ++-- tests/helpers/templeveldb.nim | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/codex/merkletree/helpers.nim b/tests/codex/merkletree/helpers.nim index 5816a12ac..848b8469e 100644 --- a/tests/codex/merkletree/helpers.nim +++ b/tests/codex/merkletree/helpers.nim @@ -5,7 +5,7 @@ import ../helpers export merkletree, helpers -converter toBool*(x: CtBool): bool = +converter toBool*(x: CTBool): bool = bool(x) proc `==`*(a, b: Poseidon2Tree): bool = diff --git a/tests/codex/node/helpers.nim b/tests/codex/node/helpers.nim index 2b73649a2..8ca067f9a 100644 --- a/tests/codex/node/helpers.nim +++ b/tests/codex/node/helpers.nim @@ -71,8 +71,8 @@ template setupAndTearDown*() {.dirty.} = network: BlockExcNetwork clock: Clock localStore: RepoStore - localStoreRepoDs: DataStore - localStoreMetaDs: DataStore + localStoreRepoDs: Datastore + localStoreMetaDs: Datastore engine: BlockExcEngine store: NetworkStore node: CodexNodeRef diff --git a/tests/helpers/templeveldb.nim b/tests/helpers/templeveldb.nim index 97b405538..054336915 100644 --- a/tests/helpers/templeveldb.nim +++ b/tests/helpers/templeveldb.nim @@ -14,9 +14,9 @@ var number = 0 proc newDb*(self: TempLevelDb): Datastore = if self.currentPath.len > 0: raiseAssert("TempLevelDb already active.") - self.currentPath = getTempDir() / "templeveldb" / $number / $getmonotime() + self.currentPath = getTempDir() / "templeveldb" / $number / $getMonoTime() inc number - createdir(self.currentPath) + createDir(self.currentPath) self.ds = LevelDbDatastore.new(self.currentPath).tryGet() return self.ds @@ -26,5 +26,5 @@ proc destroyDb*(self: TempLevelDb): Future[void] {.async.} = try: (await self.ds.close()).tryGet() finally: - removedir(self.currentPath) + removeDir(self.currentPath) self.currentPath = "" From 627e6ae5a63ddea867bcf19d6d709ed8ce30b865 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:19:44 +0100 Subject: [PATCH 21/44] Fix enums in tests --- tests/codex/sales/testsalesagent.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codex/sales/testsalesagent.nim b/tests/codex/sales/testsalesagent.nim index 215f8bb48..fe19ecb04 100644 --- a/tests/codex/sales/testsalesagent.nim +++ b/tests/codex/sales/testsalesagent.nim @@ -101,7 +101,7 @@ asyncchecksuite "Sales agent": clock.set(market.requestExpiry[request.id] + 1) check eventually onCancelCalled - for requestState in {RequestState.New, Started, Finished, Failed}: + for requestState in {RequestState.New, RequestState.Started, RequestState.Finished, RequestState.Failed}: test "onCancelled is not called when request state is " & $requestState: agent.start(MockState.new()) await agent.subscribe() @@ -110,7 +110,7 @@ asyncchecksuite "Sales agent": await sleepAsync(100.millis) check not onCancelCalled - for requestState in {RequestState.Started, Finished, Failed}: + for requestState in {RequestState.Started, RequestState.Finished, RequestState.Failed}: test "cancelled future is finished when request state is " & $requestState: agent.start(MockState.new()) await agent.subscribe() From 51a40548d1b73975fa747ed47576bbbc48cb9e4e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:19:59 +0100 Subject: [PATCH 22/44] Fix camel case --- tests/codex/slots/helpers.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codex/slots/helpers.nim b/tests/codex/slots/helpers.nim index 7459161e9..cb5170505 100644 --- a/tests/codex/slots/helpers.nim +++ b/tests/codex/slots/helpers.nim @@ -97,11 +97,11 @@ proc createProtectedManifest*( protectedTreeCid = protectedTree.rootCid().tryGet() for index, cid in cids[0.. Date: Thu, 26 Dec 2024 07:20:33 +0100 Subject: [PATCH 23/44] Extract node components to a variable to make Nim 2 happy --- tests/codex/helpers/nodeutils.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/codex/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim index 8c624bb21..42449eceb 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -46,7 +46,8 @@ proc generateNodes*( networkStore = NetworkStore.new(engine, localStore) switch.mount(network) - result.add(( + + let nc : NodesComponents = ( switch, discovery, wallet, @@ -56,7 +57,9 @@ proc generateNodes*( pendingBlocks, blockDiscovery, engine, - networkStore)) + networkStore) + + result.add(nc) proc connectNodes*(nodes: seq[Switch]) {.async.} = for dialer in nodes: From 3f9f421c74089c3102c4baae82353555b2cc425a Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 26 Dec 2024 07:21:57 +0100 Subject: [PATCH 24/44] Update the tests using ValidIpAddress to IpAddress --- tests/codex/slots/testbackendfactory.nim | 18 +++++++++--------- tests/codex/slots/testprover.nim | 6 +++--- tests/contracts/testDeployment.nim | 13 +++++++------ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim index 669fb03b2..2f96aa12e 100644 --- a/tests/codex/slots/testbackendfactory.nim +++ b/tests/codex/slots/testbackendfactory.nim @@ -47,9 +47,9 @@ suite "Test BackendFactory": let config = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), @@ -68,9 +68,9 @@ suite "Test BackendFactory": let config = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, @@ -90,9 +90,9 @@ suite "Test BackendFactory": let config = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circuitDir: OutDir(circuitDir) diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim index ea47d93ac..4f875d756 100644 --- a/tests/codex/slots/testprover.nim +++ b/tests/codex/slots/testprover.nim @@ -43,9 +43,9 @@ suite "Test Prover": metaDs = metaTmp.newDb() config = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim index f89e28a8c..55dded3a0 100644 --- a/tests/contracts/testDeployment.nim +++ b/tests/contracts/testDeployment.nim @@ -15,16 +15,17 @@ method getChainId*(provider: MockProvider): Future[UInt256] {.async: (raises:[Pr proc configFactory(): CodexConf = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1")) + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: static parseIpAddress("127.0.0.1") + ) proc configFactory(marketplace: Option[EthAddress]): CodexConf = CodexConf( cmd: StartUpCmd.persistence, - nat: ValidIpAddress.init("127.0.0.1"), - discoveryIp: ValidIpAddress.init(IPv4_any()), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + nat: static parseIpAddress("127.0.0.1"), + discoveryIp: IPv4_any(), + metricsAddress: static parseIpAddress("127.0.0.1"), marketplaceAddress: marketplace) asyncchecksuite "Deployment": From d2388a32fc7694700e007b28c7a6fdad43ff6760 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 27 Dec 2024 12:09:40 +0100 Subject: [PATCH 25/44] Fix cast for value which is already an option --- codex/utils/options.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/codex/utils/options.nim b/codex/utils/options.nim index 115af7823..ef49347a3 100644 --- a/codex/utils/options.nim +++ b/codex/utils/options.nim @@ -9,6 +9,14 @@ proc `as`*[T](value: T, U: type): ?U = ## Casts a value to another type, returns an Option. ## When the cast succeeds, the option will contain the casted value. ## When the cast fails, the option will have no value. + + # In Nim 2.0.x, check 42.some as int == none(int) + # Maybe because some 42.some looks like Option[Option[int]] + # So we check first that the value is an option of the expected type. + # In that case, we do not need to do anything, just return the value as it is. + when value is Option[U]: + return value + when value is U: return some value elif value is ref object: From d9eca3b93e1f89b2cb87b3a0c4cb276701713bc3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 2 Jan 2025 09:43:47 +0100 Subject: [PATCH 26/44] Set nim version to 2.0.x for CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d27077a3..72bcca3ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches - nim_version: pinned + nim_version: 2.0.12 concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} From 696f66d0027980bb2e3cfc5fcb89671ce89bfcef Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 2 Jan 2025 10:00:10 +0100 Subject: [PATCH 27/44] Set nim version to 2.0.x for CI --- .github/actions/nimbus-build-system/action.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index b80c05c75..52762cf05 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -11,7 +11,7 @@ inputs: default: "amd64" nim_version: description: "Nim version" - default: "version-1-6" + default: "v2.0.12" rust_version: description: "Rust version" default: "1.79.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72bcca3ef..5b83d287c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches - nim_version: 2.0.12 + nim_version: v2.0.12 concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} From 56d2186655f900a95fe3865781e4d7b6782004e9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 3 Jan 2025 17:53:09 +0100 Subject: [PATCH 28/44] Move to miniupnp version 2.2.4 to avoid symlink error --- vendor/nim-nat-traversal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 5e4059746..3398a649f 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 5e4059746e9095e1731b02eeaecd62a70fbe664d +Subproject commit 3398a649f4e895cae86fbe85b54769ebc0967878 From 1ecad686d1187deea82c07ee666b5e8a05b33338 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 6 Jan 2025 17:17:55 +0100 Subject: [PATCH 29/44] Set core.symlinks to false for Windows for miniupnp >= 2.2.5 support --- .github/actions/nimbus-build-system/action.yml | 6 ++++++ vendor/nim-nat-traversal | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 52762cf05..60efe6d86 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -168,6 +168,12 @@ runs: shell: ${{ inputs.shell }} {0} run: echo "NIM_COMMIT=${{ inputs.nim_version }}" >> ${GITHUB_ENV} + - name: MSYS2 (Windows All) - Disable git symbolic links (since miniupnp 2.2.5) + if: inputs.os == 'windows' + shell: ${{ inputs.shell }} {0} + run: | + git config --global core.symlinks false + - name: Build Nim and Codex dependencies shell: ${{ inputs.shell }} {0} run: | diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 3398a649f..5e4059746 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 3398a649f4e895cae86fbe85b54769ebc0967878 +Subproject commit 5e4059746e9095e1731b02eeaecd62a70fbe664d From 7bdbd87f48676b23eba07399fafedafa9648b9ca Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 7 Jan 2025 15:10:22 +0100 Subject: [PATCH 30/44] Update to Nim 2.0.14 --- Makefile | 2 +- tests/codex/testasyncheapqueue.nim | 9 ++++++--- vendor/nim-codex-dht | 2 +- vendor/nim-contract-abi | 2 +- vendor/nim-datastore | 2 +- vendor/nim-ethers | 2 +- vendor/nim-poseidon2 | 2 +- vendor/nimbus-build-system | 2 +- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index ce1fdaea2..22cb2b319 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ # If NIM_COMMIT is set to "nimbusbuild", this will use the # version pinned by nimbus-build-system. #PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21 -PINNED_NIM_VERSION := v2.0.12 +PINNED_NIM_VERSION := v2.0.14 ifeq ($(NIM_COMMIT),) NIM_COMMIT := $(PINNED_NIM_VERSION) else ifeq ($(NIM_COMMIT),pinned) diff --git a/tests/codex/testasyncheapqueue.nim b/tests/codex/testasyncheapqueue.nim index 10bab65d9..dcdafcf2c 100644 --- a/tests/codex/testasyncheapqueue.nim +++ b/tests/codex/testasyncheapqueue.nim @@ -163,8 +163,9 @@ asyncchecksuite "Asynchronous Tests": test "Test update": var heap = newAsyncHeapQueue[Task](5) + let data = [("a", 4), ("b", 3), ("c", 2)] - for item in [("a", 4), ("b", 3), ("c", 2)]: + for item in data: check heap.pushNoWait(item).isOk check heap[0] == (name: "c", priority: 2) @@ -173,8 +174,9 @@ asyncchecksuite "Asynchronous Tests": test "Test pushOrUpdate - update": var heap = newAsyncHeapQueue[Task](3) + let data = [("a", 4), ("b", 3), ("c", 2)] - for item in [("a", 4), ("b", 3), ("c", 2)]: + for item in data: check heap.pushNoWait(item).isOk check heap[0] == (name: "c", priority: 2) @@ -183,8 +185,9 @@ asyncchecksuite "Asynchronous Tests": test "Test pushOrUpdate - push": var heap = newAsyncHeapQueue[Task](2) + let data = [("a", 4), ("b", 3)] - for item in [("a", 4), ("b", 3)]: + for item in data: check heap.pushNoWait(item).isOk check heap[0] == ("b", 3) # sanity check for order diff --git a/vendor/nim-codex-dht b/vendor/nim-codex-dht index 5f22be042..4bd3a39e0 160000 --- a/vendor/nim-codex-dht +++ b/vendor/nim-codex-dht @@ -1 +1 @@ -Subproject commit 5f22be04208d8ead97eed0af8af47033e1211f23 +Subproject commit 4bd3a39e0030f8ee269ef217344b6b59ec2be6dc diff --git a/vendor/nim-contract-abi b/vendor/nim-contract-abi index 363f8b485..842f48910 160000 --- a/vendor/nim-contract-abi +++ b/vendor/nim-contract-abi @@ -1 +1 @@ -Subproject commit 363f8b4852b05c089b1e35ef3a0a882d51c996bc +Subproject commit 842f48910be4f388bcbf8abf1f02aba1d5e2ee64 diff --git a/vendor/nim-datastore b/vendor/nim-datastore index b5635ef82..d67860add 160000 --- a/vendor/nim-datastore +++ b/vendor/nim-datastore @@ -1 +1 @@ -Subproject commit b5635ef82f50255bc5ff2fe411722d870bcb587c +Subproject commit d67860add63fd23cdacde1d3da8f4739c2660c2d diff --git a/vendor/nim-ethers b/vendor/nim-ethers index eea5811f0..4fd75adc4 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit eea5811f00f7d72ed3b0edaa2f15332bf5382773 +Subproject commit 4fd75adc46a3a1e4bb3ef9daf629f222d8530db0 diff --git a/vendor/nim-poseidon2 b/vendor/nim-poseidon2 index f5a351134..4e2c6e619 160000 --- a/vendor/nim-poseidon2 +++ b/vendor/nim-poseidon2 @@ -1 +1 @@ -Subproject commit f5a3511349f77027e0b10cbdeb953c130cc8268c +Subproject commit 4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3 diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 8fafcd0ba..4c6ff070c 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 8fafcd0bac9f409091b7bcaee62ab6330f57441e +Subproject commit 4c6ff070c116450bb2c285691724ac9e6202cb28 From dfcdac6f254e268af10993a8a9af50bd4f5bbeab Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 7 Jan 2025 15:16:02 +0100 Subject: [PATCH 31/44] Update CI nim versions to 2.0.14 --- .github/actions/nimbus-build-system/action.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 60efe6d86..7384b22f3 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -11,7 +11,7 @@ inputs: default: "amd64" nim_version: description: "Nim version" - default: "v2.0.12" + default: "v2.0.14" rust_version: description: "Rust version" default: "1.79.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b83d287c..acc05c1ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches - nim_version: v2.0.12 + nim_version: v2.0.14 concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} From f24d2642dcc7177d3ae6428e69db1948e2779f5e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 8 Jan 2025 14:02:12 +0100 Subject: [PATCH 32/44] Try with GCC 14 --- .github/actions/nimbus-build-system/action.yml | 15 +++++++++++++-- .github/workflows/ci.yml | 8 ++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 7384b22f3..de599b418 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -78,11 +78,21 @@ runs: mingw-w64-i686-ntldd-git mingw-w64-i686-rust - - name: MSYS2 (Windows All) - Downgrade to gcc 13 + - name: MSYS2 (Windows All) - Update to gcc 14 if: inputs.os == 'windows' shell: ${{ inputs.shell }} {0} run: | - pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-13.2.0-6-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst + pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-14.2.0-2-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-14.2.0-2-any.pkg.tar.zst + + - name: Install gcc 14 on Linux + # Should be removed when ubuntu-latest is 26.04 + if : ${{ inputs.os == 'linux' }} + shell: ${{ inputs.shell }} {0} + run: | + # Add GCC-14 to alternatives + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 + # Set GCC-14 as the default + sudo update-alternatives --set gcc /usr/bin/gcc-14 - name: Derive environment variables shell: ${{ inputs.shell }} {0} @@ -177,6 +187,7 @@ runs: - name: Build Nim and Codex dependencies shell: ${{ inputs.shell }} {0} run: | + gcc --version make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update echo ./env.sh nim --version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acc05c1ef..3ac255509 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,10 @@ jobs: uses: fabiocaccamo/create-matrix-action@v4 with: matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} From fdba4e7fcdef02942f4c58fd17abdc1234921ecf Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 8 Jan 2025 14:06:49 +0100 Subject: [PATCH 33/44] Replace apt-fast by apt-get --- .github/actions/nimbus-build-system/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index de599b418..2a70e39de 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -31,8 +31,8 @@ runs: if: inputs.os == 'linux' && (inputs.cpu == 'amd64' || inputs.cpu == 'arm64') shell: ${{ inputs.shell }} {0} run: | - sudo apt-fast update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + sudo apt-get update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ --no-install-recommends -yq lcov - name: APT (Linux i386) @@ -40,8 +40,8 @@ runs: shell: ${{ inputs.shell }} {0} run: | sudo dpkg --add-architecture i386 - sudo apt-fast update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + sudo apt-get update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ --no-install-recommends -yq gcc-multilib g++-multilib - name: Homebrew (macOS) From 251d2867495076ed60478873b56935cb7017cc14 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 8 Jan 2025 15:47:37 +0100 Subject: [PATCH 34/44] Update ubuntu runner to latest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ac255509..a79c38dd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} coverage: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v4 From b5848da18821a1679524b3efc798946ffcc26c79 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 12:58:58 +0100 Subject: [PATCH 35/44] Use Ubuntu 20.04 for coverage --- .github/actions/nimbus-build-system/action.yml | 4 ++-- .github/workflows/ci-reusable.yml | 1 + .github/workflows/ci.yml | 7 ++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 2a70e39de..67ad6c986 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -85,8 +85,8 @@ runs: pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-14.2.0-2-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-14.2.0-2-any.pkg.tar.zst - name: Install gcc 14 on Linux - # Should be removed when ubuntu-latest is 26.04 - if : ${{ inputs.os == 'linux' }} + # We don't want to install gcc 14 for coverage (Ubuntu 20.04) + if : ${{ inputs.os == 'linux' && !inputs.coverage }} shell: ${{ inputs.shell }} {0} run: | # Add GCC-14 to alternatives diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 55846d898..ce66a9b66 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -40,6 +40,7 @@ jobs: os: ${{ matrix.os }} shell: ${{ matrix.shell }} nim_version: ${{ matrix.nim_version }} + coverage: false ## Part 1 Tests ## - name: Unit tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a79c38dd7..9b865de66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,11 @@ jobs: cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} coverage: - runs-on: ubuntu-latest + # Force to stick to ubuntu 20.04 for coverage because + # lcov was updated to 2.x version in ubuntu-latest + # and cause a lot of issues. + # See https://github.com/linux-test-project/lcov/issues/238 + runs-on: ubuntu-20.04 steps: - name: Checkout sources uses: actions/checkout@v4 @@ -61,6 +65,7 @@ jobs: with: os: linux nim_version: ${{ env.nim_version }} + coverage: true - name: Generate coverage data run: | From 801f6b767537a56f05bc8e65a380b52d513427b1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 13:20:48 +0100 Subject: [PATCH 36/44] Disable CI cache for coverage --- .github/actions/nimbus-build-system/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 67ad6c986..b88f96a40 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -169,6 +169,7 @@ runs: - name: Restore Nim toolchain binaries from cache id: nim-cache uses: actions/cache@v4 + if : ${{ !inputs.coverage }} with: path: NimBinaries key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_version }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} From 2c436db3f62dde49ee8a69224f55265c8862fa40 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 17:12:35 +0100 Subject: [PATCH 37/44] Add coverage property description --- .github/actions/nimbus-build-system/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index b88f96a40..c7bdb6278 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -18,6 +18,9 @@ inputs: shell: description: "Shell to run commands in" default: "bash --noprofile --norc -e -o pipefail" + coverage: + description: "True if the process is used for coverage" + default: false runs: using: "composite" steps: From 958df72d1085aa5c7a2e157c8d05feb2629e0049 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 17:15:14 +0100 Subject: [PATCH 38/44] Remove commented test --- tests/codex/testchunking.nim | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 3a8c0b4b8..216fbcc4c 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -118,13 +118,4 @@ asyncchecksuite "Chunking": test "stream should forward LPStreamError": expect LPStreamError: - await raiseStreamException(newException(LPStreamError, "test error")) - - # This test cannot exist anymore. - # The signature of the method readOnce is explicitly listing the error raised: - # LPStreamError and CancelledError. - # So trying to raise CatchableError will not compile because CatchableError is - # not in the pragma array amd Nim 2 looks stricter for this rule. - # test "stream should convert other exceptions to defect": - # expect Defect: - # await raiseStreamException(newException(CatchableError, "test error")) + await raiseStreamException(newException(LPStreamError, "test error")) \ No newline at end of file From 08ced590633671cdb4c259cd0ebcee2f317fc6c5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 17:29:11 +0100 Subject: [PATCH 39/44] Check the node value of seen instead of using alreadySeen --- codex/rest/json.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 4a0009430..afbfebe6d 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -90,7 +90,7 @@ proc init*(_: type RestNode, node: dn.Node): RestNode = peerId: node.record.data.peerId, record: node.record, address: node.address, - seen: node.alreadySeen() + seen: node.seen > 0.5 ) proc init*(_: type RestRoutingTable, routingTable: rt.RoutingTable): RestRoutingTable = From 10bc903da9cb952d5a20c81a4be62d37d3efcd56 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 19:09:51 +0100 Subject: [PATCH 40/44] Fix the merge. The taskpool work was reverted. --- codex/node.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codex/node.nim b/codex/node.nim index 49367df97..f2bdecc4b 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -250,8 +250,7 @@ proc streamEntireDataset( erasure = Erasure.new( self.networkStore, leoEncoderProvider, - leoDecoderProvider, - self.taskpool) + leoDecoderProvider) without _ =? (await erasure.decode(manifest)), error: error "Unable to erasure decode manifest", manifestCid, exc = error.msg return failure(error) From 66d1c5b493a004ce0118eed37cd9a83c30e8bbe6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 9 Jan 2025 19:10:08 +0100 Subject: [PATCH 41/44] Update nim-ethers submodule --- vendor/nim-ethers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 4fd75adc4..1cfccb969 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 4fd75adc46a3a1e4bb3ef9daf629f222d8530db0 +Subproject commit 1cfccb9695fa47860bf7ef3d75da9019096a3933 From fb14b6066fe3cff7d11ded8c3379e3faa280d320 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Jan 2025 10:31:07 +0100 Subject: [PATCH 42/44] Remove deprecated ValidIpAddress. Fix missing case and imports. --- codex/nat.nim | 34 +++++++++++++----------- codex/utils/addrutils.nim | 12 ++++----- codex/utils/natutils.nim | 12 ++++----- tests/codex/slots/testbackendfactory.nim | 7 ++--- tests/codex/slots/testprover.nim | 3 ++- tests/codex/testnat.nim | 2 +- tests/contracts/testDeployment.nim | 4 +-- 7 files changed, 39 insertions(+), 35 deletions(-) diff --git a/codex/nat.nim b/codex/nat.nim index 933dc7e8f..f4c88074d 100644 --- a/codex/nat.nim +++ b/codex/nat.nim @@ -73,6 +73,8 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] msg = "Internet Gateway Device found but it's not connected. Trying anyway." of NotAnIGD: msg = "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway." + of IGDIpNotRoutable: + msg = "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway." if not quiet: debug "UPnP", msg if canContinue: @@ -115,7 +117,7 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] # default) is a public IP. That's a long shot, because code paths involving a # user-provided bind address are not supposed to get here. proc getRoutePrefSrc( - bindIp: ValidIpAddress): (Option[ValidIpAddress], PrefSrcStatus) = + bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) = let bindAddress = initTAddress(bindIp, Port(0)) if bindAddress.isAnyLocal(): @@ -124,18 +126,18 @@ proc getRoutePrefSrc( # No route was found, log error and continue without IP. error "No routable IP address found, check your network connection", error = ip.error - return (none(ValidIpAddress), NoRoutingInfo) + return (none(IpAddress), NoRoutingInfo) elif ip.get().isGlobalUnicast(): return (some(ip.get()), PrefSrcIsPublic) else: - return (none(ValidIpAddress), PrefSrcIsPrivate) + return (none(IpAddress), PrefSrcIsPrivate) elif bindAddress.isGlobalUnicast(): - return (some(ValidIpAddress.init(bindIp)), BindAddressIsPublic) + return (some(bindIp), BindAddressIsPublic) else: - return (none(ValidIpAddress), BindAddressIsPrivate) + return (none(IpAddress), BindAddressIsPrivate) # Try to detect a public IP assigned to this host, before trying NAT traversal. -proc getPublicRoutePrefSrcOrExternalIP*(natStrategy: NatStrategy, bindIp: ValidIpAddress, quiet = true): Option[ValidIpAddress] = +proc getPublicRoutePrefSrcOrExternalIP*(natStrategy: NatStrategy, bindIp: IpAddress, quiet = true): Option[IpAddress] = let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) case prefSrcStatus: @@ -144,7 +146,7 @@ proc getPublicRoutePrefSrcOrExternalIP*(natStrategy: NatStrategy, bindIp: ValidI of PrefSrcIsPrivate, BindAddressIsPrivate: let extIp = getExternalIP(natStrategy, quiet) if extIp.isSome: - return some(ValidIpAddress.init(extIp.get)) + return some(extIp.get) proc doPortMapping(tcpPort, udpPort: Port, description: string): Option[(Port, Port)] {.gcsafe.} = var @@ -294,14 +296,14 @@ proc redirectPorts*(tcpPort, udpPort: Port, description: string): Option[(Port, proc setupNat*(natStrategy: NatStrategy, tcpPort, udpPort: Port, clientId: string): - tuple[ip: Option[ValidIpAddress], tcpPort, udpPort: Option[Port]] = + tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] = ## Setup NAT port mapping and get external IP address. ## If any of this fails, we don't return any IP address but do return the ## original ports as best effort. ## TODO: Allow for tcp or udp port mapping to be optional. let extIp = getExternalIP(natStrategy) if extIp.isSome: - let ip = ValidIpAddress.init(extIp.get) + let ip = extIp.get let extPorts = ({.gcsafe.}: redirectPorts(tcpPort = tcpPort, udpPort = udpPort, @@ -311,20 +313,20 @@ proc setupNat*(natStrategy: NatStrategy, tcpPort, udpPort: Port, (ip: some(ip), tcpPort: some(extTcpPort), udpPort: some(extUdpPort)) else: warn "UPnP/NAT-PMP available but port forwarding failed" - (ip: none(ValidIpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) + (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) else: warn "UPnP/NAT-PMP not available" - (ip: none(ValidIpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) + (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) type NatConfig* = object case hasExtIp*: bool - of true: extIp*: ValidIpAddress + of true: extIp*: IpAddress of false: nat*: NatStrategy -proc setupAddress*(natConfig: NatConfig, bindIp: ValidIpAddress, +proc setupAddress*(natConfig: NatConfig, bindIp: IpAddress, tcpPort, udpPort: Port, clientId: string): - tuple[ip: Option[ValidIpAddress], tcpPort, udpPort: Option[Port]] + tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] {.gcsafe.} = ## Set-up of the external address via any of the ways as configured in ## `NatConfig`. In case all fails an error is logged and the bind ports are @@ -353,10 +355,10 @@ proc setupAddress*(natConfig: NatConfig, bindIp: ValidIpAddress, return (prefSrcIp, some(tcpPort), some(udpPort)) of PrefSrcIsPrivate: error "No public IP address found. Should not use --nat:none option" - return (none(ValidIpAddress), some(tcpPort), some(udpPort)) + return (none(IpAddress), some(tcpPort), some(udpPort)) of BindAddressIsPrivate: error "Bind IP is not a public IP address. Should not use --nat:none option" - return (none(ValidIpAddress), some(tcpPort), some(udpPort)) + return (none(IpAddress), some(tcpPort), some(udpPort)) of NatStrategy.NatUpnp, NatStrategy.NatPmp: return setupNat(natConfig.nat, tcpPort, udpPort, clientId) diff --git a/codex/utils/addrutils.nim b/codex/utils/addrutils.nim index 587c1710a..3eec3015c 100644 --- a/codex/utils/addrutils.nim +++ b/codex/utils/addrutils.nim @@ -41,7 +41,7 @@ func remapAddr*( MultiAddress.init(parts.join("/")) .expect("Should construct multiaddress") -proc getMultiAddrWithIPAndUDPPort*(ip: ValidIpAddress, port: Port): MultiAddress = +proc getMultiAddrWithIPAndUDPPort*(ip: IpAddress, port: Port): MultiAddress = ## Creates a MultiAddress with the specified IP address and UDP port ## ## Parameters: @@ -54,7 +54,7 @@ proc getMultiAddrWithIPAndUDPPort*(ip: ValidIpAddress, port: Port): MultiAddress let ipFamily = if ip.family == IpAddressFamily.IPv4: "/ip4/" else: "/ip6/" return MultiAddress.init(ipFamily & $ip & "/udp/" & $port).expect("valid multiaddr") -proc getAddressAndPort*(ma: MultiAddress): tuple[ip: Option[ValidIpAddress], port: Option[Port]] = +proc getAddressAndPort*(ma: MultiAddress): tuple[ip: Option[IpAddress], port: Option[Port]] = try: # Try IPv4 first let ipv4Result = ma[multiCodec("ip4")] @@ -63,7 +63,7 @@ proc getAddressAndPort*(ma: MultiAddress): tuple[ip: Option[ValidIpAddress], por .protoArgument() .expect("Invalid IPv4 format") let ipArray = [ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3]] - some(ipv4(ipArray)) + some(IpAddress(family: IPv4, address_v4: ipArray)) else: # Try IPv6 if IPv4 not found let ipv6Result = ma[multiCodec("ip6")] @@ -74,9 +74,9 @@ proc getAddressAndPort*(ma: MultiAddress): tuple[ip: Option[ValidIpAddress], por var ipArray: array[16, byte] for i in 0..15: ipArray[i] = ipBytes[i] - some(ipv6(ipArray)) + some(IpAddress(family: IPv6, address_v6: ipArray)) else: - none(ValidIpAddress) + none(IpAddress) # Get TCP Port let portResult = ma[multiCodec("tcp")] @@ -89,4 +89,4 @@ proc getAddressAndPort*(ma: MultiAddress): tuple[ip: Option[ValidIpAddress], por none(Port) (ip: ip, port: port) except Exception: - (ip: none(ValidIpAddress), port: none(Port)) + (ip: none(IpAddress), port: none(Port)) diff --git a/codex/utils/natutils.nim b/codex/utils/natutils.nim index 25f083bdf..86497e12b 100644 --- a/codex/utils/natutils.nim +++ b/codex/utils/natutils.nim @@ -16,14 +16,14 @@ type type IpLimits* = object limit*: uint - ips: Table[ValidIpAddress, uint] + ips: Table[IpAddress, uint] -func hash*(ip: ValidIpAddress): Hash = +func hash*(ip: IpAddress): Hash = case ip.family of IpAddressFamily.IPv6: hash(ip.address_v6) of IpAddressFamily.IPv4: hash(ip.address_v4) -func inc*(ipLimits: var IpLimits, ip: ValidIpAddress): bool = +func inc*(ipLimits: var IpLimits, ip: IpAddress): bool = let val = ipLimits.ips.getOrDefault(ip, 0) if val < ipLimits.limit: ipLimits.ips[ip] = val + 1 @@ -31,7 +31,7 @@ func inc*(ipLimits: var IpLimits, ip: ValidIpAddress): bool = else: false -func dec*(ipLimits: var IpLimits, ip: ValidIpAddress) = +func dec*(ipLimits: var IpLimits, ip: IpAddress) = let val = ipLimits.ips.getOrDefault(ip, 0) if val == 1: ipLimits.ips.del(ip) @@ -48,7 +48,7 @@ func isGlobalUnicast*(address: IpAddress): bool = let a = initTAddress(address, Port(0)) a.isGlobalUnicast() -proc getRouteIpv4*(): Result[ValidIpAddress, cstring] = +proc getRouteIpv4*(): Result[IpAddress, cstring] = # Avoiding Exception with initTAddress and can't make it work with static. # Note: `publicAddress` is only used an "example" IP to find the best route, # no data is send over the network to this IP! @@ -65,4 +65,4 @@ proc getRouteIpv4*(): Result[ValidIpAddress, cstring] = # This should not occur really. error "Address conversion error", exception = e.name, msg = e.msg return err("Invalid IP address") - ok(ValidIpAddress.init(ip)) \ No newline at end of file + ok(ip) \ No newline at end of file diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim index 2889fc82f..fb832024d 100644 --- a/tests/codex/slots/testbackendfactory.nim +++ b/tests/codex/slots/testbackendfactory.nim @@ -7,6 +7,7 @@ import pkg/codex/conf import pkg/codex/slots/proofs/backends import pkg/codex/slots/proofs/backendfactory import pkg/codex/slots/proofs/backendutils +import pkg/codex/utils/natutils import ../helpers import ../examples @@ -50,7 +51,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), @@ -72,7 +73,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, @@ -95,7 +96,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circuitDir: OutDir(circuitDir) diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim index 2b6f5d855..08098a8ca 100644 --- a/tests/codex/slots/testprover.nim +++ b/tests/codex/slots/testprover.nim @@ -13,6 +13,7 @@ import pkg/confutils/defs import pkg/poseidon2/io import pkg/codex/utils/poseidon2digest import pkg/codex/nat +import pkg/codex/utils/natutils import ./helpers import ../helpers @@ -38,7 +39,7 @@ suite "Test Prover": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + metricsAddress: static parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), diff --git a/tests/codex/testnat.nim b/tests/codex/testnat.nim index 1930a7c5d..f6c5bb8e9 100644 --- a/tests/codex/testnat.nim +++ b/tests/codex/testnat.nim @@ -15,7 +15,7 @@ suite "NAT Address Tests": udpPort = Port(1234) natConfig = NatConfig( hasExtIp: true, - extIp:ValidIpAddress.init("8.8.8.8")) + extIp: static parseIpAddress("8.8.8.8")) # Create test addresses localAddr = MultiAddress.init("/ip4/127.0.0.1/tcp/5000").expect("valid multiaddr") diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim index 665829e6d..dbe7362a5 100644 --- a/tests/contracts/testDeployment.nim +++ b/tests/contracts/testDeployment.nim @@ -18,7 +18,7 @@ proc configFactory(): CodexConf = nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1")) + metricsAddress: IpAddress("127.0.0.1")) proc configFactory(marketplace: Option[EthAddress]): CodexConf = CodexConf( @@ -26,7 +26,7 @@ proc configFactory(marketplace: Option[EthAddress]): CodexConf = nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: ValidIpAddress.init("127.0.0.1"), + metricsAddress: static parseIpAddress("127.0.0.1"), marketplaceAddress: marketplace) asyncchecksuite "Deployment": From 5cbb43bd7ebed4d1a8cfc914877c47b036a6fda6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Jan 2025 10:31:39 +0100 Subject: [PATCH 43/44] Fix a weird issue where nim-confutils cannot find NatAny --- codex/conf.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/codex/conf.nim b/codex/conf.nim index 70cae5a28..41ee628e2 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -43,6 +43,7 @@ import ./units import ./utils import ./nat import ./utils/natutils + from ./validationconfig import MaxSlots, ValidationGroups export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig @@ -147,7 +148,7 @@ type nat* {. desc: "Specify method to use for determining public address. " & "Must be one of: any, none, upnp, pmp, extip:" - defaultValue: NatConfig(hasExtIp: false, nat: NatAny) + defaultValue: defaultNatConfig() defaultValueDesc: "any" name: "nat" }: NatConfig @@ -413,6 +414,9 @@ logutils.formatIt(LogFormat.json, EthAddress): %it func defaultAddress*(conf: CodexConf): IpAddress = result = static parseIpAddress("127.0.0.1") +func defaultNatConfig*(): NatConfig = + result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) + func persistence*(self: CodexConf): bool = self.cmd == StartUpCmd.persistence @@ -486,7 +490,7 @@ func parseCmdArg*(T: type NatConfig, p: string): T {.raises: [ValueError].} = else: if p.startsWith("extip:"): try: - let ip = ValidIpAddress.init(p[6..^1]) + let ip = parseIpAddress(p[6..^1]) NatConfig(hasExtIp: true, extIp: ip) except ValueError: let error = "Not a valid IP address: " & p[6..^1] From 0d71b0a664242a67935230afa9fa71b9fe872e06 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Jan 2025 10:58:50 +0100 Subject: [PATCH 44/44] Fix tests and remove useless static keyword --- tests/codex/slots/testbackendfactory.nim | 6 +++--- tests/codex/slots/testprover.nim | 2 +- tests/codex/testnat.nim | 2 +- tests/contracts/testDeployment.nim | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim index fb832024d..87a7733a1 100644 --- a/tests/codex/slots/testbackendfactory.nim +++ b/tests/codex/slots/testbackendfactory.nim @@ -51,7 +51,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: static parseIpAddress("127.0.0.1"), + metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), @@ -73,7 +73,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: static parseIpAddress("127.0.0.1"), + metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, @@ -96,7 +96,7 @@ suite "Test BackendFactory": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: static parseIpAddress("127.0.0.1"), + metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, circuitDir: OutDir(circuitDir) diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim index 08098a8ca..f6deaebac 100644 --- a/tests/codex/slots/testprover.nim +++ b/tests/codex/slots/testprover.nim @@ -39,7 +39,7 @@ suite "Test Prover": nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: static parseIpAddress("127.0.0.1"), + metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), diff --git a/tests/codex/testnat.nim b/tests/codex/testnat.nim index f6c5bb8e9..c9bccfec4 100644 --- a/tests/codex/testnat.nim +++ b/tests/codex/testnat.nim @@ -15,7 +15,7 @@ suite "NAT Address Tests": udpPort = Port(1234) natConfig = NatConfig( hasExtIp: true, - extIp: static parseIpAddress("8.8.8.8")) + extIp: parseIpAddress("8.8.8.8")) # Create test addresses localAddr = MultiAddress.init("/ip4/127.0.0.1/tcp/5000").expect("valid multiaddr") diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim index dbe7362a5..49f01774e 100644 --- a/tests/contracts/testDeployment.nim +++ b/tests/contracts/testDeployment.nim @@ -2,6 +2,7 @@ import pkg/ethers import codex/contracts/deployment import codex/conf import codex/contracts +import pkg/codex/utils/natutils import ../asynctest import ../checktest @@ -18,7 +19,7 @@ proc configFactory(): CodexConf = nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: IpAddress("127.0.0.1")) + metricsAddress: parseIpAddress("127.0.0.1")) proc configFactory(marketplace: Option[EthAddress]): CodexConf = CodexConf( @@ -26,7 +27,7 @@ proc configFactory(marketplace: Option[EthAddress]): CodexConf = nat: NatConfig( hasExtIp: false, nat: NatNone), - metricsAddress: static parseIpAddress("127.0.0.1"), + metricsAddress: parseIpAddress("127.0.0.1"), marketplaceAddress: marketplace) asyncchecksuite "Deployment":