update dial timeout, ensure K found nodes (#2083)

* update dial timeout, ensure K found nodes * add libp2p config variables to node config * add network module to default test config * bump store protocol version * Updated logs for removing file on path * Updated logs for removing file on path Co-authored-by: Djordje Kovacevic <[email protected]>
OriginTrail · Sep 9, 2022 · a81d60e · a81d60e
1 parent b13c8dc
commit a81d60e
Show file tree

Hide file tree

Showing 9 changed files with 121 additions and 26 deletions.
diff --git a/config/config.json b/config/config.json
@@ -36,6 +36,19 @@
                     "libp2p-service": {
                         "package": "./network/implementation/libp2p-service.js",
                         "config": {
+                            "kBucketSize": 20,
+                            "connectionManager": {
+                                "autoDial": true,
+                                "autoDialInterval": 10e3,
+                                "dialTimeout": 2e3
+                            },
+                            "peerRouting": {
+                                "refreshManager": {
+                                    "enabled": true,
+                                    "interval": 6e5,
+                                    "bootDelay": 2e3
+                                }
+                            },
                             "port": 9000,
                             "bootstrap": []
                         }
@@ -181,6 +194,33 @@
                     }
                 }
             },
+            "network": {
+                "enabled": true,
+                "implementation": {
+                    "libp2p-service": {
+                        "package": "./network/implementation/libp2p-service.js",
+                        "config": {
+                            "kBucketSize": 20,
+                            "connectionManager": {
+                                "autoDial": true,
+                                "autoDialInterval": 10e3,
+                                "dialTimeout": 2e3
+                            },
+                            "peerRouting": {
+                                "refreshManager": {
+                                    "enabled": true,
+                                    "interval": 6e5,
+                                    "bootDelay": 2e3
+                                }
+                            },
+                            "port": 9000,
+                            "bootstrap": [
+                                "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj"
+                            ]
+                        }
+                    }
+                }
+            },
             "repository": {
                 "enabled": true,
                 "implementation": {
@@ -286,6 +326,19 @@
                     "libp2p-service": {
                         "package": "./network/implementation/libp2p-service.js",
                         "config": {
+                            "kBucketSize": 20,
+                            "connectionManager": {
+                                "autoDial": true,
+                                "autoDialInterval": 10e3,
+                                "dialTimeout": 2e3
+                            },
+                            "peerRouting": {
+                                "refreshManager": {
+                                    "enabled": true,
+                                    "interval": 6e5,
+                                    "bootDelay": 2e3
+                                }
+                            },
                             "port": 9000,
                             "bootstrap": [
                                 "/ip4/139.59.174.24/tcp/9000/p2p/QmU7p61f8qgzCtfTEUynJaqnsLRN4QExjtWfsMRQ7oiMuS",

diff --git a/dependencies.md b/dependencies.md
@@ -218,6 +218,10 @@
 - **version**: ^2.1.3
 - **description**: convert expiration time to milliseconds in `token-generation.js`
 
+##### [multiformats](https://www.npmjs.com/package/multiformats)
+-   **version**: ^9.8.1
+-   **description**: calculate sha256 hash of peerIds (libp2p find nodes operation)
+
 ##### [mysql2](https://www.npmjs.com/package/mysql2) 
 - **version**: ^2.3.3
 - **description**: 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -94,6 +94,7 @@
         "libp2p": "^0.38.0",
         "merkletreejs": "^0.2.32",
         "ms": "^2.1.3",
+        "multiformats": "^9.8.1",
         "mysql2": "^2.3.3",
         "p-iteration": "^1.1.8",
         "pino": "^8.4.2",

diff --git a/src/constants/constants.js b/src/constants/constants.js
@@ -221,7 +221,7 @@ export const COMMAND_STATUS = {
  *  Network protocols
  */
 export const NETWORK_PROTOCOLS = {
-    STORE: '/store/1.0.0',
+    STORE: '/store/1.0.1',
     GET: '/get/1.0.0',
     SEARCH: '/search/1.0.0',
 };

diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js
@@ -1,5 +1,6 @@
 /* eslint-disable import/no-unresolved */
 import { createLibp2p } from 'libp2p';
+import { sha256 } from 'multiformats/hashes/sha2';
 import { Bootstrap } from '@libp2p/bootstrap';
 import { Mplex } from '@libp2p/mplex';
 import { Noise } from '@chainsafe/libp2p-noise';
@@ -10,6 +11,8 @@ import * as lp from 'it-length-prefixed';
 import { unmarshalPrivateKey } from '@libp2p/crypto/keys';
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string';
+import { xor as uint8ArrayXor } from 'uint8arrays/xor';
+import { compare as uint8ArrayCompare } from 'uint8arrays/compare';
 import map from 'it-map';
 import { createFromPrivKey, createRSAPeerId } from '@libp2p/peer-id-factory';
 import { InMemoryRateLimiter } from 'rolling-rate-limiter';
@@ -29,14 +32,17 @@ const initializationObject = {
     transports: [new TCP()],
     streamMuxers: [new Mplex()],
     connectionEncryption: [new Noise()],
-    dht: new KadDHT(),
 };
 
 class Libp2pService {
     async initialize(config, logger) {
         this.config = config;
         this.logger = logger;
 
+        initializationObject.dht = new KadDHT({ kBucketSize: this.config.kBucketSize });
+        initializationObject.peerRouting = this.config.peerRouting;
+        initializationObject.connectionManager = this.config.connectionManager;
+
         if (this.config.bootstrap.length > 0) {
             initializationObject.peerDiscovery = [
                 new Bootstrap({
@@ -87,9 +93,6 @@ class Libp2pService {
     }
 
     _initializeNodeListeners() {
-        this.node.addEventListener('peer:discovery', (peer) => {
-            this._onPeerDiscovery(peer);
-        });
         this.node.connectionManager.addEventListener('peer:connect', (connection) => {
             this._onPeerConnect(connection);
         });
@@ -114,12 +117,6 @@ class Libp2pService {
         this.blackList = {};
     }
 
-    _onPeerDiscovery(peer) {
-        this.logger.trace(
-            `Node ${this.node.peerId.toString()} discovered ${peer.detail.id.toString()}`,
-        );
-    }
-
     _onPeerConnect(connection) {
         this.logger.trace(
             `Node ${this.node.peerId.toString()} connected to ${connection.detail.remotePeer.toString()}`,
@@ -152,12 +149,22 @@ class Libp2pService {
             },
         );
 
-        const nodes = [];
+        const keyHash = Buffer.from((await sha256.digest(encodedKey)).digest);
+        const unsortedPeerDistances = [];
         for await (const finalPeerId of finalPeerIds) {
-            nodes.push(finalPeerId);
+            const peerHash = Buffer.from((await sha256.digest(finalPeerId.toBytes())).digest);
+
+            unsortedPeerDistances.push({
+                peerId: finalPeerId,
+                distance: uint8ArrayXor(keyHash, peerHash),
+            });
         }
 
-        return nodes;
+        const sortedPeers = unsortedPeerDistances
+            .sort((a, b) => uint8ArrayCompare(a.distance, b.distance))
+            .map((pd) => pd.peerId);
+
+        return sortedPeers.slice(0, this.config.kBucketSize);
     }
 
     getPeers() {

diff --git a/src/service/file-service.js b/src/service/file-service.js
@@ -64,7 +64,7 @@ class FileService {
     }
 
     async removeFile(filePath) {
-        this.logger.debug(`Removing file on path: ${filePath}`);
+        this.logger.trace(`Removing file on path: ${filePath}`);
         await unlink(filePath);
         return true;
     }
@@ -107,9 +107,9 @@ class FileService {
             if (createdDate.getTime() + expiredTimeout < now.getTime()) {
                 // eslint-disable-next-line no-await-in-loop
                 await this.removeFile(filePath);
-                this.logger.trace(`Successfully removed expired cache file: ${filePath}`);
             }
         }
+        this.logger.trace(`Successfully removed ${fileList.length} expired cache files`);
     }
 }
 

diff --git a/test/bdd/steps/config/origintrail-test-bootstrap-config.json b/test/bdd/steps/config/origintrail-test-bootstrap-config.json
@@ -25,9 +25,22 @@
         "libp2p-service": {
           "package": "./network/implementation/libp2p-service.js",
           "config": {
-            "port": 9000,
-            "bootstrap": [],
-            "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ="
+              "kBucketSize": 20,
+              "connectionManager": {
+                  "autoDial": true,
+                  "autoDialInterval": 10e3,
+                  "dialTimeout": 2e3
+              },
+              "peerRouting": {
+                  "refreshManager": {
+                      "enabled": true,
+                      "interval": 6e5,
+                      "bootDelay": 2e3
+                  }
+              },
+              "port": 9000,
+              "bootstrap": [],
+              "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ="
           }
         }
       }

diff --git a/test/bdd/steps/config/origintrail-test-node-config.json b/test/bdd/steps/config/origintrail-test-node-config.json
@@ -25,10 +25,23 @@
         "libp2p-service": {
           "package": "./network/implementation/libp2p-service.js",
           "config": {
-            "port": 9001,
-            "bootstrap": [
-              "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj"
-            ]
+              "kBucketSize": 20,
+              "connectionManager": {
+                  "autoDial": true,
+                  "autoDialInterval": 10e3,
+                  "dialTimeout": 2e3
+              },
+              "peerRouting": {
+                  "refreshManager": {
+                      "enabled": true,
+                      "interval": 6e5,
+                      "bootDelay": 2e3
+                  }
+              },
+              "port": 9001,
+              "bootstrap": [
+                  "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj"
+              ]
           }
         }
       }