StableLM-3B-4E1T-Completion-Function.json

[
    {
        "id": "f9aafb1df135391b",
        "type": "function",
        "z": "706bdd3f3f20c07c",
        "name": "StableLM Completion",
        "func": "(async () => {\n  let Prompt = (msg.payload || '').trim()\n  if (Prompt === '') {\n    msg.payload = 'empty or missing prompt'\n    node.send([null, msg])\n    node.done()\n    return\n  }\n\n  /**** retrieve settings or provide defaults ****/\n\n  let Seed = parseInt(msg.seed, 10)\n  if (isNaN(Seed)) { Seed = -1 }\n\n  let Threads = parseInt(msg.threads, 10)\n  if (isNaN(Threads)) { Threads = 4 }\n  Threads = Math.max(1, Threads)\n  Threads = Math.min(Threads, Math.max(1, os.cpus().length))\n\n  let ContextLength = parseInt(msg.context, 10)\n  if (isNaN(ContextLength)) { ContextLength = 512 }\n  ContextLength = Math.max(0, Math.min(ContextLength, 4096))\n\n  let keep = parseInt(msg.keep, 10)\n  if (isNaN(keep)) { keep = 0 }\n  keep = Math.max(-1, Math.min(keep, ContextLength))\n\n  let PredictionLength = parseInt(msg.predict, 10)\n  if (isNaN(PredictionLength)) { PredictionLength = 128 }\n  PredictionLength = Math.max(1, Math.min(PredictionLength, ContextLength)) // no -1!\n\n  let topK = parseInt(msg.topk, 10)\n  if (isNaN(topK)) { topK = 40 }\n  topK = Math.max(1, Math.min(topK, 100))\n\n  let topP = parseFloat(msg.topp)\n  if (isNaN(topP)) { topP = 0.9 }\n  topP = Math.max(0.1, Math.min(topP, 1.0))\n\n  let Temperature = parseFloat(msg.temperature)\n  if (isNaN(Temperature)) { Temperature = 0.8 }\n  Temperature = Math.max(0.0, Math.min(Temperature, 1.0))\n\n  let Batches = parseInt(msg.batches, 10)\n  if (isNaN(Batches)) { Batches = 8 }\n  Batches = Math.max(1, Math.min(Batches, 100))\n\n  let Grammar = (msg.grammar || '').trim()\n\n  Prompt = Prompt.replace(/\"/g, '\\\\\"')\n\n  /**** retrieve UserDir - crash if not a folder ****/\n\n  let UserDir = (global.get('UserDir') || '').trim()\n  if (UserDir === '') { UserDir = process.env.HOME + '/.node-red' }\n\n  if (!fs.statSync(UserDir, { throwIfNoEntry: false })?.isDirectory()) {\n    throw new Error(\n      'the given \"UserDir\" (\"' + UserDir + '\") is either missing or ' +\n      'not a folder - exiting'\n    )\n  }\n\n  /**** combine all these settings into a command ****/\n\n  const Command = (\n    './llama --model ./stablelm-3b-4e1t-Q8_0.bin --mlock ' +\n    ' --ctx_size ' + ContextLength + ' --keep ' + keep +\n    ' --n_predict ' + PredictionLength +\n    ' --threads ' + Threads + ' --batch_size ' + Batches +\n    ' --seed ' + Seed + ' --temp ' + Temperature +\n    ' --top_k ' + topK + ' --top_p ' + topP +\n    ' --reverse-prompt \"<|endoftext|>\"' +\n    (Grammar === '' ? '' : ' --grammar \"' + encoded(Grammar) + '\"') +\n    ' --prompt \"' + encoded(Prompt) + '\"'\n  )\n  const Options = {\n    cwd: UserDir\n  }\n  node.warn('Command = ' + Command)\n  /**** extract actual reponse from command output ****/\n\n  function ResponseFrom(Text) {\n    let HeaderLength = Text.indexOf('\\n\\n\\n')\n    Text = Text.slice(HeaderLength + 1)\n\n    let TrailerIndex = Text.indexOf('<|endoftext|>')\n    if (TrailerIndex < 0) {\n      TrailerIndex = Text.indexOf('\\nllama_print_timings')\n    }\n    Text = Text.slice(0, TrailerIndex)\n\n    return Text\n  }\n\n  /**** now infer a response from the given prompt ****/\n\n  node.status({ fill: 'yellow', shape: 'ring', text: 'running' })\n\n  const ShellProcess = child_process.exec(Command, Options)\n  let stdout = ''\n  for await (const Chunk of ShellProcess.stdout) {\n    stdout += Chunk\n  }\n\n  let stderr = ''\n  for await (const Chunk of ShellProcess.stderr) {\n    stderr += Chunk\n  }\n\n  const ExitCode = (await new Promise((resolve) => {\n    ShellProcess.on('close', resolve)\n  })) || 0\n  if (ExitCode !== 0) {\n    node.status({ fill: 'red', shape: 'dot', text: 'ExitCode = ' + ExitCode })\n\n    msg.statusCode = 500 + ExitCode\n    msg.payload = stderr\n  } else {\n    node.status({ fill: 'green', shape: 'dot', text: 'finished' })\n\n    stdout = stdout.trim()\n\n    msg.statusCode = (stdout === '' ? 204 : 200)\n    msg.payload = ResponseFrom(stdout)\n  }\n\n  node.send([msg, null])\n  node.done()\n\n  /**** encoded ****/\n\n  function encoded(Text) {\n    return Text.replace(/'/g, \"'\\\"'\\\"'\")\n  }\n})()\n",
        "outputs": 2,
        "timeout": "",
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "libs": [
            {
                "var": "child_process",
                "module": "child_process"
            },
            {
                "var": "fs",
                "module": "fs"
            },
            {
                "var": "process",
                "module": "process"
            },
            {
                "var": "path",
                "module": "path"
            },
            {
                "var": "os",
                "module": "os"
            }
        ],
        "x": 340,
        "y": 220,
        "wires": [
            [
                "10ee081f81a9a211",
                "50138c7c51f06e4d"
            ],
            [
                "b566f55c47395bd8"
            ]
        ],
        "outputLabels": [
            "on success",
            "on failure"
        ]
    }
]