nodejs nym-wasm-client working, needs polishing

Signed-off-by: Sebastian Martinez <[email protected]>

sdk/typescript/packages/nodejs-client/internal/index.mjs

@@ -1,24 +1,28 @@
-import { createNymMixnetClient } from '../dist/cjs-full-fat/index.cjs';
-
-debugger;
+import { createNymMixnetClient } from '../dist/cjs/index.js';
 
 const nym = await createNymMixnetClient();
 
-const nymApiUrl = 'https://validator.nymtech.net/api';
-// show message payload content when received
-nym.events.subscribeToTextMessageReceivedEvent((e) => {
-  console.log('Got a message: ', e.args.payload);
+nym.events.subscribeToLoaded((e) => {
+  console.log('Loaded');
+});
+
+nym.events.subscribeToTextMessageReceivedEvent(async (e) => {
+  console.log(e.args.payload);
+  if (e.args.payload === 'Hello') await nym.client.stop();
 });
 
+const nymApiUrl = 'https://validator.nymtech.net/api/';
+
 // start the client and connect to a gateway
 await nym.client.start({
-  clientId: 'My awesome client',
   nymApiUrl,
+  clientId: 'my-client',
 });
 
-// send a message to yourself
-const payload = 'Hello mixnet';
-const recipient = await nym.client.selfAddress();
-console.log(recipient)
-debugger;
-await nym.client.send({ payload, recipient });
+nym.events.subscribeToConnected(async (e) => {
+  console.log('Connected');
+  // send a message to yourself
+  const message = 'Hello';
+  const recipient = await nym.client.selfAddress();
+  await nym.client.send({ payload: { message, mimeType: 'text/plain' }, recipient });
+});

sdk/typescript/packages/nodejs-client/package.json

@@ -27,8 +27,11 @@
     "tsc": "tsc --noEmit true"
   },
   "dependencies": {
-    "@nymproject/nym-client-wasm-node": ">=1.2.0-rc.10 || ^1",
-    "comlink": "^4.3.1"
+    "@nymproject/nym-client-wasm-node": "^1.2.0",
+    "comlink": "^4.3.1",
+    "fake-indexeddb": "^4.0.2",
+    "rollup-plugin-polyfill": "^4.2.0",
+    "ws": "^8.14.2"
   },
   "devDependencies": {
     "@babel/core": "^7.15.0",
@@ -68,13 +71,13 @@
     "rimraf": "^3.0.2",
     "rollup": "^3.9.1",
     "rollup-plugin-base64": "^1.0.1",
+    "rollup-plugin-modify": "^3.0.0",
     "rollup-plugin-web-worker-loader": "^1.6.1",
     "ts-jest": "^29.1.0",
     "ts-loader": "^9.4.2",
     "typedoc": "^0.24.8",
     "typescript": "^4.8.4"
   },
   "private": false,
-  "type": "module",
   "types": "./dist/index.d.ts"
 }

sdk/typescript/packages/nodejs-client/rollup/cjs.mjs

@@ -17,7 +17,7 @@ export const getConfig = (opts) => ({
     format: 'cjs',
   },
   plugins: [
-    webWorkerLoader({ targetPlatform: 'node', inline: opts.inline }),
+    webWorkerLoader({ targetPlatform: 'node', inline: opts.inline, external: [], loadPath: "../dist/cjs/" }),
     resolve({ browser: false, extensions: ['.js', '.ts'] }),
     wasm({ maxFileSize: 10000000, targetEnv: 'node' }),
     typescript({

sdk/typescript/packages/nodejs-client/rollup/worker.mjs

@@ -1,11 +1,8 @@
 import typescript from '@rollup/plugin-typescript';
 import resolve from '@rollup/plugin-node-resolve';
 import { wasm } from '@rollup/plugin-wasm';
-import replace from '@rollup/plugin-replace';
 import commonjs from '@rollup/plugin-commonjs';
-import path from 'path';
-
-console.log(process.cwd());
+import modify from 'rollup-plugin-modify';
 
 /**
  * Configure worker output
@@ -15,24 +12,32 @@ console.log(process.cwd());
  */
 export const getConfig = (opts) => ({
   input: 'src/worker.ts',
-  external: ['util'],
   output: {
-    dir: 'dist',
+    dir: 'dist/cjs',
     format: 'cjs',
+    manualChunks: {
+      ws: ["ws"],
+    }
   },
+  external: ['util', "fake-indexeddb"],
   plugins: [
     resolve({
-      rootDir: process.cwd(),
       browser: false,
       preferBuiltins: true,
       extensions: ['.js', '.ts'],
     }),
     commonjs(),
-    // this is some nasty monkey patching that removes the WASM URL (because it is handled by the `wasm` plugin)
-    replace({
-      values: { "input = new URL('nym_client_wasm_bg.wasm', import.meta.url);": 'input = undefined;' },
-      delimiters: ['', ''],
-      preventAssignment: true,
+    // TODO: One of the wasm functions calls `new WebSocket` at one point, which we aren't able to polyfill correctly yet.
+    modify({ find: 'const ret = new WebSocket(getStringFromWasm0(arg0, arg1));', replace: 'var WebSocket = require("ws"); const ret = new WebSocket(getStringFromWasm0(arg0, arg1));' }),
+
+    // TODO: `getObject(...).require` seems to generate a warning on Webpack but with Rollup we get a panick since it can't require.
+    // By hard coding the require here, we can workaround that.
+    // Reference: https://github.com/rust-random/getrandom/issues/224
+    modify({ find: 'getObject(arg0).require(getStringFromWasm0(arg1, arg2));', replace: 'require("crypto");' }),
+    // TODO: The NodeJS setTimeout returns a Timeout object instead of a timeout id as the web api one does, check how we could polyfill this.
+    modify({
+      find: /const ret = getObject\(arg0\).setTimeout\(getObject\(arg1\), arg2\);\n\s*?_assertNum\((.*?)\)/,
+      replace: (match) => match.replace('_assertNum(ret)', '// _assertNum(ret)'),
     }),
     opts?.inlineWasm === true
       ? wasm({ maxFileSize: 10_000_000, targetEnv: 'node' }) // force the wasm plugin to embed the wasm bundle - this means no downstream bundlers have to worry about handling it
@@ -42,9 +47,15 @@ export const getConfig = (opts) => ({
         }),
     typescript({
       compilerOptions: {
+        outDir: 'dist/cjs',
         declaration: false,
         target: 'es5',
+
       },
     }),
+    // modify({
+    //   find: "  var worker_threads = require('worker_threads');",
+    //   replace: "  var worker_threads = require('worker_threads');\nvar WebSocket = require('ws');",
+    // }),
   ],
 });

sdk/typescript/packages/nodejs-client/scripts/build.sh

@@ -6,13 +6,53 @@ set -o pipefail
 
 rm -rf dist || true
 
+#-------------------------------------------------------
+# WEB WORKER (mix-fetch WASM)
+#-------------------------------------------------------
+# The web worker needs to be bundled because the WASM bundle needs to be loaded synchronously and all dependencies
+# must be included in the worker script (because it is not loaded as an ES Module)
+
+# build the worker
+rollup -c rollup-worker.config.mjs
+
+# move it next to the Typescript `src/index.ts` so it can be inlined by rollup
+rm -f src/*.js
+mv dist/cjs/worker.js src/worker.js
+
+# move WASM files out of build area
+# mkdir -p dist/worker
+# mv dist/*.wasm dist/worker
+
+#-------------------------------------------------------
+# COMMON JS
+#-------------------------------------------------------
+# Some old build systems cannot fully handle ESM or ES2021, so build
+# a CommonJS bundle targeting ES5
+
+# build the SDK as a CommonJS bundle
+rollup -c rollup-cjs.config.mjs
+
+# move WASM files into place
+cp src/*.wasm dist/cjs
+
+# move WASM files into place
+# cp dist/worker/*.wasm dist/cjs
+# cp dist/worker/*.js dist/cjs
+
 #-------------------------------------------------------
 # WEB WORKER (full-fat)
 #-------------------------------------------------------
 
 # build the worker
 rollup -c rollup-worker-full-fat.config.mjs
-mv dist/worker.js src/worker.js
+
+# move it next to the Typescript `src/index.ts` so it can be inlined by rollup
+# rm -f src/*.js
+# rm -f src/*.wasm
+mv dist/cjs/worker.js src/worker.js
+
+# move WASM files out of build area
+# mkdir -p dist/worker
 
 #-------------------------------------------------------
 # COMMON JS (full-fat)
@@ -22,17 +62,15 @@ mv dist/worker.js src/worker.js
 
 # build the SDK as a CommonJS bundle (with worker inlined as a blob)
 rollup -c rollup-cjs-full-fat.config.mjs
-mv dist/cjs-full-fat/index.js dist/cjs-full-fat/index.cjs
 
 #-------------------------------------------------------
 # CLEAN UP
 #-------------------------------------------------------
 
-# rm -rf dist/cjs/worker
-# rm -rf dist/cjs-full-fat/worker
+rm -rf dist/worker
 
 # copy README
 # cp README.md dist/cjs/README.md
-cp README.md dist/cjs-full-fat/README.md
+# cp README.md dist/cjs-full-fat/README.md
 
 

sdk/typescript/packages/nodejs-client/src/index.ts

@@ -70,10 +70,10 @@ export const createNymMixnetClient = async (options?: NymMixnetClientOptions): P
 
   // set any options
   if (options?.autoConvertStringMimeTypes) {
-    await client.setTextMimeTypes(options.autoConvertStringMimeTypes);
+    client.setTextMimeTypes(options.autoConvertStringMimeTypes);
   } else {
     // set some sensible defaults for text mime types
-    await client.setTextMimeTypes([MimeTypes.ApplicationJson, MimeTypes.TextPlain]);
+    client.setTextMimeTypes([MimeTypes.ApplicationJson, MimeTypes.TextPlain]);
   }
 
   // pass the client interop and subscription manage back to the caller

sdk/typescript/packages/nodejs-client/src/worker.ts

@@ -1,9 +1,23 @@
-/**
- * NB: URL syntax is used so that bundlers like webpack can load this package's code when inside the final bundle
- * the files from ../../../../nym-client-wasm will be copied into the dist directory of this package, so all import
- * paths are _relative to the output directory_ of this package (`dist`) - don't get confused!
- */
+// eslint-disable-next-line import/first
 import * as Comlink from 'comlink';
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as process from 'node:process';
+import { WebSocket } from 'ws';
+
+// polyfill setup
+const globalVar =
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval, no-restricted-globals, no-nested-ternary
+  typeof WorkerGlobalScope !== 'undefined' ? self : typeof global !== 'undefined' ? global : Function('return this;')();
+
+globalVar.WebSocket = WebSocket;
+globalVar.indexedDB = indexedDB;
+globalVar.fs = fs;
+globalVar.process = process;
+globalVar.performance = performance;
+globalVar.TextEncoder = TextEncoder;
+globalVar.TextDecoder = TextDecoder;
+globalVar.crypto = crypto;
 
 import {
   ClientConfig,
@@ -14,8 +28,8 @@ import {
   parse_utf8_string,
   utf8_string_to_byte_array,
 } from '@nymproject/nym-client-wasm-node';
-
-import nodeEndpoint from './node-adapter';
+import { indexedDB } from 'fake-indexeddb';
+import { parentPort } from 'worker_threads';
 
 import type {
   BinaryMessageReceivedEvent,
@@ -27,12 +41,10 @@ import type {
   RawMessageReceivedEvent,
   StringMessageReceivedEvent,
 } from './types';
+
+import nodeEndpoint from './node-adapter';
 import { EventKinds, MimeTypes } from './types';
-import { parentPort } from 'worker_threads';
 
-// web workers are only allowed to load external scripts as the load
-// importScripts(new URL('./nym_client_wasm.js', import.meta.url));
-// eslint-disable-next-line import/extensions
 
 console.log('[Nym WASM client] Starting Nym WASM web worker...');
 
@@ -42,13 +54,7 @@ console.log('[Nym WASM client] Starting Nym WASM web worker...');
  * see https://nodejs.org/api/worker_threads.html#workerparentport
  */
 // eslint-disable-next-line no-restricted-globals
-const postMessageWithType = <E>(event: E) => {
-  if (parentPort) {
-    return parentPort.postMessage(event);
-  } else {
-    console.error('Not on worker thread');
-  }
-};
+const postMessageWithType = <E>(event: E) => parentPort?.postMessage(event);
 
 /**
  * This class holds the state of the Nym WASM client and provides any interop needed.
@@ -147,8 +153,6 @@ class ClientWrapper {
   };
 }
 
-// sets up better stack traces in case of in-rust panics
-// importResult.set_panic_hook();
 // this wrapper handles any state that the wasm-pack interop needs, e.g. holding an instance of the instantiated WASM code
 const wrapper = new ClientWrapper();
 

sdk/typescript/packages/nodejs-client/tsconfig.json

@@ -3,9 +3,6 @@
   "compilerOptions": {
     "lib": [
       "es2021",
-      "dom",
-      "dom.iterable",
-      "esnext",
       "webworker"
     ],
     "outDir": "./dist/",
@@ -16,27 +13,18 @@
     "moduleResolution": "node",
     "skipLibCheck": true,
     "skipDefaultLibCheck": true,
+    "allowSyntheticDefaultImports": true,
     "declaration": true,
     "baseUrl": "."
   },
   "include": [
     "src/**/*.ts"
   ],
   "exclude": [
-    "jest.config.js",
-    "webpack.config.js",
-    "webpack.prod.js",
-    "webpack.common.js",
     "node_modules",
     "**/node_modules",
     "dist",
     "**/dist",
     "scripts",
-    "jest",
-    "__tests__",
-    "**/__tests__",
-    "__jest__",
-    "**/__jest__",
-    "config/*",
   ]
 }