Merge pull request #291 from Exabyte-io/chore/SOF-7227

Chore/sof 7227: build both JS and PY modules with JS runtime to be consistent

.github/workflows/cicd.yml

@@ -40,9 +40,9 @@ jobs:
     strategy:
       matrix:
         python-version:
+          # Enable 3.10 after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
           - 3.8.6
           - 3.9.x
-          # Enable after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
           - 3.10.x
           - 3.11.x
           # Enable after: AttributeError: module 'pkgutil' has no attribute 'ImpImporter'. Did you mean: 'zipimporter'?

.husky/pre-commit

@@ -5,10 +5,8 @@ SRC_PATTERN="\.json$"
 if git diff --cached --name-only | grep --quiet -E "$SRC_PATTERN"
 then
   echo "JSON assets changed. Running build scripts."
-  echo "Re-building JS assets."
-  npm run build:assets
-  echo "Re-building Python assets: requires virtual environment and dependencies (pip install .'[tests]')."
-  python build_schemas.py
+  echo "Re-building JS and PY assets using JS script."
+  npm run build:js-and-python-modules
 fi
 
 npx lint-staged

build_schemas.js

@@ -8,8 +8,10 @@ const path = require("path");
 const mergeAllOf = require("json-schema-merge-allof");
 const { ESSE } = require("./lib/js/esse");
 
+// JS Modules
+
 const esse = new ESSE();
-const { schemas, wrappedExamples } = esse;
+const { schemas, wrappedExamples, propertiesManifest, results } = esse;
 const schema = esse.buildGlobalSchema();
 
 fs.writeFileSync(
@@ -20,19 +22,47 @@ fs.writeFileSync(
 
 fs.writeFileSync("./schema.js", "module.exports = " + JSON.stringify(schema), "utf8");
 
+if (process.env.BUILD_PYTHON_MODULES === "true") {
+    // PY Modules
+    fs.writeFileSync(
+        "./src/py/mat3ra/esse/data/examples.py",
+        ["import json", `EXAMPLES = json.loads(r'''${JSON.stringify(wrappedExamples)}''')`].join(
+            "\n",
+        ),
+        "utf8",
+    );
+    fs.writeFileSync(
+        "./src/py/mat3ra/esse/data/schemas.py",
+        ["import json", `SCHEMAS = json.loads(r'''${JSON.stringify(schemas)}''')`].join("\n"),
+        "utf8",
+    );
+    fs.writeFileSync(
+        "./src/py/mat3ra/esse/data/properties.py",
+        [
+            "import json",
+            `PROPERTIES_MANIFEST = json.loads(r'''${JSON.stringify(propertiesManifest)}''')`,
+            `RESULTS = json.loads(r'''${JSON.stringify(results)}''')`,
+        ].join("\n"),
+        "utf8",
+    );
+}
+
 if (process.env.BUILD_ASSETS !== "true") {
     process.exit(0);
 }
 
 const subfolder = process.env.BUILD_PATH || "./docs/js/";
 schemas.forEach((s) => {
+    let mergedSchema = s;
     if (process.env.SKIP_MERGE_ALLOF !== "true") {
-        s = mergeAllOf(s, { resolvers: { defaultResolver: mergeAllOf.options.resolvers.title } });
+        mergedSchema = mergeAllOf(s, {
+            resolvers: { defaultResolver: mergeAllOf.options.resolvers.title },
+        });
     }
-    const id_as_path = s.$id.replace("-", "_");
+    const id_as_path = mergedSchema.$id.replace("-", "_");
     const full_path = `${subfolder}/schema/${id_as_path}.json`;
     fs.mkdirSync(path.dirname(full_path), { recursive: true });
-    fs.writeFileSync(full_path, JSON.stringify(s, null, 4), "utf8");
+    fs.writeFileSync(full_path, JSON.stringify(mergedSchema, null, 4), "utf8");
 });
 wrappedExamples.forEach((e) => {
     const id_as_path = e.path.replace("-", "_");

package.json

@@ -7,6 +7,7 @@
         "transpile": "mkdir -p lib; babel src/js --out-dir lib/js; node build_schemas.js",
         "transpile-and-build-assets": "mkdir -p lib; babel src/js --out-dir lib/js; BUILD_ASSETS=true BUILD_PATH='./lib/js' node build_schemas.js",
         "build:assets": "node build_schemas.js",
+        "build:js-and-python-modules": "BUILD_PYTHON_MODULES=true node build_schemas.js",
         "build:assets-with-docs": "BUILD_ASSETS=true node build_schemas.js",
         "test": "nyc --reporter=text mocha --bail --require @babel/register tests/js/",
         "lint": "eslint src/js && prettier --write src/js",

src/js/esse/index.js

@@ -1,17 +1,27 @@
 import Ajv from "ajv";
+import fs from "fs";
+import yaml from "js-yaml";
 
 import { buildSchemaDefinitions } from "./schemaUtils";
-import { EXAMPLES_DIR, SCHEMAS_DIR } from "./settings";
+import { EXAMPLES_DIR, PROPERTIES_MANIFEST_PATH, SCHEMAS_DIR } from "./settings";
 import { parseIncludeReferenceStatementsByDir } from "./utils";
 
 const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR);
 const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true);
+const PROPERTIES_MANIFEST = yaml.load(
+    fs.readFileSync(PROPERTIES_MANIFEST_PATH, { encoding: "utf-8" }),
+);
+const RESULTS = Object.entries(PROPERTIES_MANIFEST)
+    .map((k) => (k[1].isResult ? k[0] : null))
+    .filter((x) => x);
 
 export class ESSE {
-    constructor() {
-        this.schemas = SCHEMAS;
-        this.wrappedExamples = EXAMPLES;
-        this.examples = EXAMPLES.map((example) => example.data);
+    constructor(config = {}) {
+        this.schemas = config.schemas || SCHEMAS;
+        this.wrappedExamples = config.wrappedExamples || EXAMPLES;
+        this.examples = this.wrappedExamples.map((example) => example.data);
+        this.propertiesManifest = config.propertiesManifest || PROPERTIES_MANIFEST;
+        this.results = config.results || RESULTS;
     }
 
     getSchemaById(schemaId) {

src/js/esse/settings.js

@@ -2,3 +2,7 @@ import path from "path";
 
 export const SCHEMAS_DIR = path.resolve(__dirname, "../../../schema");
 export const EXAMPLES_DIR = path.resolve(__dirname, "../../../example");
+export const PROPERTIES_MANIFEST_PATH = path.resolve(
+    __dirname,
+    "../../../manifest/properties.yaml",
+);

src/py/mat3ra/esse/data/properties.py

@@ -1,3 +1,3 @@
 import json
-PROPERTIES_MANIFEST = json.loads(json.dumps({'convergence_ionic': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/workflow/convergence/ionic', 'isMonitor': True}, 'convergence_electronic': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/workflow/convergence/electronic', 'isMonitor': True}, 'pressure': {'defaults': {'units': 'kbar'}, 'schemaId': 'properties-directory/scalar/pressure', 'isResult': True}, 'total_energy': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/total-energy', 'isResult': True}, 'surface_energy': {'defaults': {'units': 'eV/A^2'}, 'schemaId': 'properties-directory/scalar/surface-energy', 'isResult': True}, 'fermi_energy': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/fermi-energy', 'isResult': True}, 'total_force': {'defaults': {'units': 'eV/angstrom'}, 'schemaId': 'properties-directory/scalar/total-force', 'isResult': True}, 'zero_point_energy': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/zero-point-energy', 'isResult': True}, 'reaction_energy_barrier': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/reaction-energy-barrier', 'isResult': True}, 'electron_affinity': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/electron-affinity', 'isResult': True}, 'ionization_potential': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/ionization-potential', 'isResult': True}, 'valence_band_offset': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/scalar/valence-band-offset'}, 'phonon_dos': {'defaults': {'xAxis': {'label': 'frequency', 'units': 'cm-1'}, 'yAxis': {'label': 'Phonon DOS', 'units': 'states/cm-1'}}, 'schemaId': 'properties-directory/non-scalar/phonon-dos', 'isResult': True}, 'phonon_dispersions': {'defaults': {'xAxis': {'label': 'qpoints', 'units': 'crystal'}, 'yAxis': {'label': 'frequency', 'units': 'cm-1'}}, 'schemaId': 'properties-directory/non-scalar/phonon-dispersions', 'isResult': True}, 'total_energy_contributions': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/non-scalar/total-energy-contributions', 'isResult': True}, 'band_gaps': {'types': ['direct', 'indirect'], 'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/non-scalar/band-gaps', 'isResult': True}, 'stress_tensor': {'defaults': {'units': 'kbar'}, 'schemaId': 'properties-directory/non-scalar/stress-tensor', 'isResult': True}, 'band_structure': {'defaults': {'xAxis': {'label': 'kpoints', 'units': 'crystal'}, 'yAxis': {'label': 'energy', 'units': 'eV'}}, 'schemaId': 'properties-directory/non-scalar/band-structure', 'isResult': True}, 'density_of_states': {'defaults': {'xAxis': {'label': 'energy', 'units': 'eV'}, 'yAxis': {'label': 'density of states', 'units': 'states/unitcell'}}, 'schemaId': 'properties-directory/non-scalar/density-of-states', 'isResult': True}, 'reaction_energy_profile': {'defaults': {'xAxis': {'label': 'reaction coordinate'}, 'yAxis': {'label': 'energy', 'units': 'eV'}}, 'schemaId': 'properties-directory/non-scalar/reaction-energy-profile', 'isResult': True}, 'potential_profile': {'defaults': {'xAxis': {'label': 'z coordinate'}, 'yAxis': {'label': 'energy', 'units': 'eV'}}, 'schemaId': 'properties-directory/non-scalar/potential-profile', 'isResult': True}, 'charge_density_profile': {'defaults': {'xAxis': {'label': 'z coordinate'}, 'yAxis': {'label': 'charge density', 'units': 'e/A'}}, 'schemaId': 'properties-directory/non-scalar/charge-density-profile', 'isResult': True}, 'vibrational_spectrum': {'defaults': {'xAxis': {'label': 'wavenumber', 'units': 'cm-1'}, 'yAxis': {'label': 'Absorption coefficient', 'units': 'km/mol'}}, 'schemaId': 'properties-directory/non-scalar/vibrational-spectrum', 'isResult': True}, 'file_content': {'schemaId': 'properties-directory/non-scalar/file-content', 'isResult': True}, 'average_potential_profile': {'defaults': {'xAxis': {'label': 'z coordinate', 'units': 'angstrom'}, 'yAxis': {'label': 'energy', 'units': 'eV'}}, 'schemaId': 'properties-directory/non-scalar/average-potential-profile', 'isResult': True}, 'dielectric_tensor': {'schemaId': 'properties-directory/non-scalar/dielectric-tensor', 'isResult': True}, 'basis': {'defaults': {'units': 'angstrom'}, 'schemaId': 'properties-directory/structural/basis'}, 'lattice': {'defaults': {'units': 'angstrom'}, 'schemaId': 'properties-directory/structural/lattice'}, 'atomic_forces': {'defaults': {'units': 'eV/angstrom'}, 'schemaId': 'properties-directory/structural/atomic-forces', 'isResult': True}, 'atomic_constraints': {'schemaId': 'properties-directory/structural/basis/atomic-constraints'}, 'p-norm': {'schemaId': 'properties-directory/structural/p-norm'}, 'volume': {'defaults': {'units': 'angstrom^3'}, 'schemaId': 'properties-directory/structural/volume'}, 'symmetry': {'defaults': {'units': 'angstrom'}, 'schemaId': 'properties-directory/structural/symmetry'}, 'elemental_ratio': {'schemaId': 'properties-directory/structural/elemental-ratio'}, 'density': {'defaults': {'units': 'g/cm^3'}, 'schemaId': 'properties-directory/structural/density'}, 'magnetic_moments': {'defaults': {'units': 'uB'}, 'schemaId': 'properties-directory/structural/magnetic-moments', 'isResult': True}, 'bonds': {'defaults': {'units': 'angstrom'}, 'schemaId': 'properties-directory/structural/basis/bonds'}, 'inchi': {'schemaId': 'properties-directory/structural/inchi'}, 'inchi_key': {'schemaId': 'properties-directory/structural/inchi-key'}, 'material': {'schemaId': 'material'}, 'hubbard_u': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/non-scalar/hubbard-u', 'isResult': True}, 'hubbard_v': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/non-scalar/hubbard-v', 'isResult': True}, 'hubbard_v_nn': {'defaults': {'units': 'eV'}, 'schemaId': 'properties-directory/non-scalar/hubbard-v-nn', 'isResult': True}}))
-RESULTS = json.loads(json.dumps(['pressure', 'total_energy', 'surface_energy', 'fermi_energy', 'total_force', 'zero_point_energy', 'reaction_energy_barrier', 'electron_affinity', 'ionization_potential', 'phonon_dos', 'phonon_dispersions', 'total_energy_contributions', 'band_gaps', 'stress_tensor', 'band_structure', 'density_of_states', 'reaction_energy_profile', 'potential_profile', 'charge_density_profile', 'vibrational_spectrum', 'file_content', 'average_potential_profile', 'dielectric_tensor', 'atomic_forces', 'magnetic_moments', 'hubbard_u', 'hubbard_v', 'hubbard_v_nn']))
+PROPERTIES_MANIFEST = json.loads(r'''{"convergence_ionic":{"defaults":{"units":"eV"},"schemaId":"properties-directory/workflow/convergence/ionic","isMonitor":true},"convergence_electronic":{"defaults":{"units":"eV"},"schemaId":"properties-directory/workflow/convergence/electronic","isMonitor":true},"pressure":{"defaults":{"units":"kbar"},"schemaId":"properties-directory/scalar/pressure","isResult":true},"total_energy":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/total-energy","isResult":true},"surface_energy":{"defaults":{"units":"eV/A^2"},"schemaId":"properties-directory/scalar/surface-energy","isResult":true},"fermi_energy":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/fermi-energy","isResult":true},"total_force":{"defaults":{"units":"eV/angstrom"},"schemaId":"properties-directory/scalar/total-force","isResult":true},"zero_point_energy":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/zero-point-energy","isResult":true},"reaction_energy_barrier":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/reaction-energy-barrier","isResult":true},"electron_affinity":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/electron-affinity","isResult":true},"ionization_potential":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/ionization-potential","isResult":true},"valence_band_offset":{"defaults":{"units":"eV"},"schemaId":"properties-directory/scalar/valence-band-offset"},"phonon_dos":{"defaults":{"xAxis":{"label":"frequency","units":"cm-1"},"yAxis":{"label":"Phonon DOS","units":"states/cm-1"}},"schemaId":"properties-directory/non-scalar/phonon-dos","isResult":true},"phonon_dispersions":{"defaults":{"xAxis":{"label":"qpoints","units":"crystal"},"yAxis":{"label":"frequency","units":"cm-1"}},"schemaId":"properties-directory/non-scalar/phonon-dispersions","isResult":true},"total_energy_contributions":{"defaults":{"units":"eV"},"schemaId":"properties-directory/non-scalar/total-energy-contributions","isResult":true},"band_gaps":{"types":["direct","indirect"],"defaults":{"units":"eV"},"schemaId":"properties-directory/non-scalar/band-gaps","isResult":true},"stress_tensor":{"defaults":{"units":"kbar"},"schemaId":"properties-directory/non-scalar/stress-tensor","isResult":true},"band_structure":{"defaults":{"xAxis":{"label":"kpoints","units":"crystal"},"yAxis":{"label":"energy","units":"eV"}},"schemaId":"properties-directory/non-scalar/band-structure","isResult":true},"density_of_states":{"defaults":{"xAxis":{"label":"energy","units":"eV"},"yAxis":{"label":"density of states","units":"states/unitcell"}},"schemaId":"properties-directory/non-scalar/density-of-states","isResult":true},"reaction_energy_profile":{"defaults":{"xAxis":{"label":"reaction coordinate"},"yAxis":{"label":"energy","units":"eV"}},"schemaId":"properties-directory/non-scalar/reaction-energy-profile","isResult":true},"potential_profile":{"defaults":{"xAxis":{"label":"z coordinate"},"yAxis":{"label":"energy","units":"eV"}},"schemaId":"properties-directory/non-scalar/potential-profile","isResult":true},"charge_density_profile":{"defaults":{"xAxis":{"label":"z coordinate"},"yAxis":{"label":"charge density","units":"e/A"}},"schemaId":"properties-directory/non-scalar/charge-density-profile","isResult":true},"vibrational_spectrum":{"defaults":{"xAxis":{"label":"wavenumber","units":"cm-1"},"yAxis":{"label":"Absorption coefficient","units":"km/mol"}},"schemaId":"properties-directory/non-scalar/vibrational-spectrum","isResult":true},"file_content":{"schemaId":"properties-directory/non-scalar/file-content","isResult":true},"average_potential_profile":{"defaults":{"xAxis":{"label":"z coordinate","units":"angstrom"},"yAxis":{"label":"energy","units":"eV"}},"schemaId":"properties-directory/non-scalar/average-potential-profile","isResult":true},"dielectric_tensor":{"schemaId":"properties-directory/non-scalar/dielectric-tensor","isResult":true},"basis":{"defaults":{"units":"angstrom"},"schemaId":"properties-directory/structural/basis"},"lattice":{"defaults":{"units":"angstrom"},"schemaId":"properties-directory/structural/lattice"},"atomic_forces":{"defaults":{"units":"eV/angstrom"},"schemaId":"properties-directory/structural/atomic-forces","isResult":true},"atomic_constraints":{"schemaId":"properties-directory/structural/basis/atomic-constraints"},"p-norm":{"schemaId":"properties-directory/structural/p-norm"},"volume":{"defaults":{"units":"angstrom^3"},"schemaId":"properties-directory/structural/volume"},"symmetry":{"defaults":{"units":"angstrom"},"schemaId":"properties-directory/structural/symmetry"},"elemental_ratio":{"schemaId":"properties-directory/structural/elemental-ratio"},"density":{"defaults":{"units":"g/cm^3"},"schemaId":"properties-directory/structural/density"},"magnetic_moments":{"defaults":{"units":"uB"},"schemaId":"properties-directory/structural/magnetic-moments","isResult":true},"bonds":{"defaults":{"units":"angstrom"},"schemaId":"properties-directory/structural/basis/bonds"},"inchi":{"schemaId":"properties-directory/structural/inchi"},"inchi_key":{"schemaId":"properties-directory/structural/inchi-key"},"material":{"schemaId":"material"},"hubbard_u":{"defaults":{"units":"eV"},"schemaId":"properties-directory/non-scalar/hubbard-u","isResult":true},"hubbard_v":{"defaults":{"units":"eV"},"schemaId":"properties-directory/non-scalar/hubbard-v","isResult":true},"hubbard_v_nn":{"defaults":{"units":"eV"},"schemaId":"properties-directory/non-scalar/hubbard-v-nn","isResult":true}}''')
+RESULTS = json.loads(r'''["pressure","total_energy","surface_energy","fermi_energy","total_force","zero_point_energy","reaction_energy_barrier","electron_affinity","ionization_potential","phonon_dos","phonon_dispersions","total_energy_contributions","band_gaps","stress_tensor","band_structure","density_of_states","reaction_energy_profile","potential_profile","charge_density_profile","vibrational_spectrum","file_content","average_potential_profile","dielectric_tensor","atomic_forces","magnetic_moments","hubbard_u","hubbard_v","hubbard_v_nn"]''')