Merge pull request #410 from dehilsterlexis/NLS_SCRIPT_UPDATES

fix: update nls scripts

.vscode/launch.json

@@ -102,8 +102,17 @@
             "webRoot": "${workspaceRoot}"
         },
         {
-            "name": "NLS Util",
-            "program": "${workspaceFolder}/lib-util/index.js",
+            "name": "NLS Generate",
+            "program": "${workspaceFolder}/lib-util/generate.js",
+            "request": "launch",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "pwa-node"
+        },
+        {
+            "name": "NLS Merge",
+            "program": "${workspaceFolder}/lib-util/merge.js",
             "request": "launch",
             "skipFiles": [
                 "<node_internals>/**"

package.json

@@ -18,7 +18,8 @@
     "clean": "rimraf --glob out lib* dist types hpcc-systems.ecl.vsix",
     "copy-resources": "copyfiles -f ./node_modules/@hpcc-js/ddl-shim/schema/v2.json ./dist",
     "gen-grammar": "wsl -e ./scripts/grammar-generate.sh",
-    "gen-nls": "node ./lib-util/index.js",
+    "gen-nls": "node ./lib-util/generate.js",
+    "merge-nls": "node ./lib-util/merge.js",
     "compile": "tsc",
     "compile-watch": "npm run compile -- -watch",
     "compile-es6": "tsc --module es2020 --outDir ./lib-es6",
@@ -45,6 +46,7 @@
   },
   "dependencies": {
     "@vscode/webview-ui-toolkit": "1.2.2",
+    "child-process": "^1.0.2",
     "csv": "^6.3.5",
     "exceljs": "^4.4.0"
   },
@@ -740,15 +742,15 @@
       },
       {
         "command": "notebook.cell.name",
-        "title": "Result Name",
+        "title": "%Result Name%",
         "icon": {
           "dark": "./resources/dark/string.svg",
           "light": "./resources/light/string.svg"
         }
       },
       {
         "command": "notebook.cell.db",
-        "title": "Database Name",
+        "title": "%Database Name%",
         "icon": {
           "dark": "./resources/dark/server-process.svg",
           "light": "./resources/light/server-process.svg"

util/generate.ts

@@ -0,0 +1,5 @@
+import { fixPackage, updateLocaleFiles, googleTrans } from "./nls/vscode";
+
+fixPackage();
+updateLocaleFiles();
+googleTrans();

util/googtrans.py

@@ -0,0 +1,54 @@
+import os
+import json 
+import re
+import googletrans
+import codecs
+
+# pip install googletrans==3.1.0a0
+
+translator = googletrans.Translator()
+# print(googletrans.LANGUAGES)  
+
+directory_path = "./tmp"
+
+def sortUnique(json_object):
+    if json_object and len(json_object):
+        sorted_json = {k: json_object[k] for k in sorted(json_object, key=lambda x: x.lower())}
+        return sorted_json
+
+def googleTranslate(text, dest):
+    translation = translator.translate(text, dest=dest)
+
+    # https://stackoverflow.com/questions/63073788/google-translate-api-returns-non-utf8-characters
+    translatedText = codecs.escape_decode(translation.text)[0]
+    translatedText = translatedText.decode("utf8")
+    # print(f"Translated: {dest} {translatedText}")
+    return translatedText
+
+def translateFile(file_name, dest):
+    with open(file_name, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    new_data = {}
+    for str in data:
+        new_data[str] = googleTranslate(str, dest)
+
+    sorted_data = sortUnique(new_data)
+
+    new_file = file_name.replace(".missing.", ".trans.")
+    with open(new_file, "w", encoding="utf-8") as f:
+        json.dump(sorted_data, f, indent=4, ensure_ascii=False)
+    print(f"Translated: {new_file}")
+
+pattern = r'package\.nls\.([a-z\-]+)\.missing' 
+
+for file_name in os.listdir("./tmp"):
+    if file_name.startswith("package.nls."):
+        matches = re.findall(pattern, file_name)
+        if matches:
+            dest = matches[0]
+            if dest == "pt-br":
+                dest = "pt"
+            elif dest == "zh":
+                dest = "zh-cn"
+            translateFile(os.path.join(directory_path, file_name), dest);

util/merge.ts

@@ -0,0 +1,3 @@
+import { mergeLocales } from "./nls/vscode";
+
+mergeLocales();

util/nls/util.ts

@@ -16,7 +16,7 @@ export function encodeKey(id: string): string {
 }
 
 export function sortAndWrite(filePath: string, obj: object) {
-    const keys = Object.keys(obj).sort();
+    const keys = Object.keys(obj).sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base" }));
     const tmp = {};
     keys.forEach(k => tmp[k] = obj[k]);
     fs.writeFileSync(filePath, JSON.stringify(tmp, undefined, 4), "utf-8");

util/nls/vscode.ts

@@ -1,13 +1,33 @@
 import * as fs from "fs";
 import * as path from "path";
-
+import * as child from "child_process";
 import { sortAndWrite } from "./util";
 
-const locales = ["bs", "es", "fr", "hr", "hu", "pt-br", "sr", "zh"];
+const locales = ["es", "fr", "pt-br", "zh"]; // "bs", "es", "fr", "hr", "hu", "pt-br", "sr", "zh"
+const prevWeeks = ["Last Week", "Two Weeks Ago", "Three Weeks Ago", "Four Weeks Ago", "Five Weeks Ago", "Six Weeks Ago", "Seven Weeks Ago"];
+const tmpDir = "./tmp/";
 
 const pkg = JSON.parse(fs.readFileSync("./package.json", "utf-8"));
 const en = JSON.parse(fs.readFileSync("./package.nls.json", "utf-8"));
 
+export function initLocales() {
+    prevWeeks.forEach( (week) => {
+        if (en[week] === undefined) {
+            en[week] = week;
+        }
+    });
+}
+
+export function googleTrans() {
+    runPythonScript("./util/googtrans.py")
+    .then((output) => {
+        console.log("Google translation complete");
+    })
+    .catch((error) => {
+        console.error(`Error executing Python script: ${error.message}`);
+    }); 
+}
+
 function fix(item: any, what: string): boolean {
     let key = item[what];
     if (key === undefined) return false;
@@ -51,7 +71,7 @@ export function fixPackage() {
 
     if (touched) {
         fs.writeFileSync("./package.json", JSON.stringify(pkg, undefined, 2), "utf-8");
-        sortAndWrite("./package.nls.json", en);
+        sortAndWrite(tmpDir + "package.nls.missing.json", en);
     }
 }
 
@@ -77,13 +97,40 @@ function traverseDir(dir, found: object) {
     });
 }
 
+async function runPythonScript(scriptPath: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const pythonProcess = child.spawn("python", [scriptPath]);
+
+        let stdoutData = "";
+        let stderrData = "";
+
+        pythonProcess.stdout.on("data", (data) => {
+            stdoutData += data.toString();
+        });
+
+        pythonProcess.stderr.on("data", (data) => {
+            stderrData += data.toString();
+        });
+
+        pythonProcess.on("exit", (code) => {
+            if (code === 0) {
+                resolve(stdoutData.trim());
+            } else {
+                reject(new Error(`Python process exited with code ${code}. Error: ${stderrData.trim()}`));
+            }
+        });
+    });
+}
+
 export function updateLocaleFiles() {
+    initLocales();
     const found = {};
+
     traverseDir("./src", found);
 
     locales.forEach(l => {
         const otherFile = `./package.nls.${l}.json`;
-        const otherFileMissing = `./tmp/package.nls.${l}.missing.json`;
+        const otherFileMissing = `${tmpDir}package.nls.${l}.missing.json`;
         let other = {};
         if (fs.existsSync(otherFile)) {
             other = JSON.parse(fs.readFileSync(otherFile, "utf-8"));
@@ -113,3 +160,45 @@ export function updateLocaleFiles() {
     });
 }
 
+function mergeLocal(origFile: string, missingFile: string): void {
+
+    if (!fs.existsSync(path.join("./", origFile))) {
+        console.log(`Original file not found: ${origFile}`);
+        return;
+    }
+
+    if (!fs.existsSync(path.join(tmpDir, missingFile))) {
+        console.log(`Translation file not found: ${missingFile}`);
+        return;
+    }
+
+    let currentData: any;
+    let newData: any;
+    try {
+        currentData = JSON.parse(fs.readFileSync(path.join("./", origFile), "utf-8"));
+        newData = JSON.parse(fs.readFileSync(path.join(tmpDir, missingFile), "utf-8"));
+    } catch (err) {
+        console.error(`Error reading files: ${err}`);
+        return;
+    }
+
+    for (const fieldName in newData) {
+        if (!(fieldName in currentData)) {
+            currentData[fieldName] = newData[fieldName];
+        }
+    }
+
+    const mergedFile: string = path.join(tmpDir, origFile);
+    sortAndWrite(mergedFile, currentData);
+}
+
+export function mergeLocales() {
+    for (const fileName of fs.readdirSync(tmpDir)) {
+        if (fileName.startsWith("package.nls.") && fileName.includes(".trans.")) {
+            mergeLocal(fileName.replace(".trans.", "."), fileName);
+        } else if (fileName == "package.nls.missing.json") {
+            mergeLocal(fileName.replace(".missing.", "."), fileName);
+        }
+    }
+}
+