Merge branch 'main' into vraymond/magit

ml4ai · Nov 1, 2023 · 86db3be · 86db3be
2 parents f4fd956 + 4a526d6
commit 86db3be
Show file tree

Hide file tree

Showing 7 changed files with 321 additions and 241 deletions.
diff --git a/Dockerfile.skema-py b/Dockerfile.skema-py
@@ -5,18 +5,25 @@ FROM  python:3.8-bullseye
 # Install prerequisites
 # ======================
 ARG DEBIAN_FRONTEND=noninteractive
+ARG NODE_MAJOR=18
+
 RUN apt-get update &&\
     apt-get -y --no-install-recommends install \
         tree \
         # Required for pygraphviz
         build-essential \
         graphviz \
         libgraphviz-dev \
-        python3-venv 
+        python3-venv \
+        gnupg \
+        ca-certificates \
+        curl
 
 # Node needed for img2mml
-RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - &&\
-apt-get install -y nodejs
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get -y install nodejs
 
 # The two commands below are to reduce the size of the Docker image
 RUN apt-get clean &&\

diff --git a/pyproject.toml b/pyproject.toml
@@ -72,7 +72,7 @@ metal = ["skema[ml]",
       "askem_extractions[all]@git+https://github.com/ml4ai/ASKEM-TA1-DataModel"]
 
 # for llm use in skema
-llms = ["langchain==0.0.325"]
+llms = ["langchain==0.0.325", "openai==0.28.0"]
 
 # dependencies for text reading utilities.
 tr = ["skema[ml]", "pyarrow==13.0.0",

diff --git a/skema/rest/utils.py b/skema/rest/utils.py
@@ -18,11 +18,29 @@ def fn_preprocessor(function_network: Dict[str, Any]):
     2) metadata being inline for bf entries instead of an index into the metadata_collection -> which we will replace with an index of 2
     3) missing function_type field on a bf entry -> will replace with function_type: "IMPORTED"
     4) If there is not a body field to a function -> replace "FUNCTION" with "ABSTRACT and set "name":"unknown"
-    5) NOT DONE YET: In the future we will preprocess about function calls being arguments, in order to simplify extracting the dataflow 
+    5) If there are -1 entries in the metadata for line spans and col spans -> replaced with 1
+    6) NOT DONE YET: In the future we will preprocess about function calls being arguments, in order to simplify extracting the dataflow 
     '''
 
     # first we check the top bf level of wires and inline metadata: 
     keys_to_check = ['bf', 'wff', 'wfopi', 'wfopo', 'wopio']
+    metadata_keys_to_check = ['line_begin', 'line_end', 'col_begin', 'col_end']
+    for key in metadata_keys_to_check:
+        try: 
+            for (i, entry) in enumerate(fn_data['modules'][0]['metadata_collection']):
+                try:
+                    for (j, datum) in enumerate(entry):
+                        try:
+                            if datum[key] == -1:
+                                datum[key] = 1
+                                logs.append(f"The {j + 1}'th metadata in the {i+1} metadata index has -1 for the {key} entry")
+                        except:
+                            continue
+                except:
+                    continue
+        except:
+            continue
+
     for key in keys_to_check:
         if key == 'bf':
             try:
@@ -50,10 +68,16 @@ def fn_preprocessor(function_network: Dict[str, Any]):
                 continue
         else:
             try:
-                for (i, entry) in enumerate(fn_data['modules'][0]['fn'][key]):
-                    if entry['tgt'] == -1:
-                        del fn_data['modules'][0]['fn'][key][i]
-                        logs.append(f"The {i + 1}'th {key} wire in the top level bf is targeting -1")
+                for (i, entry) in enumerate(reversed(fn_data['modules'][0]['fn'][key])):
+                    try:
+                        if entry['tgt'] == -1:
+                            try:
+                                fn_data['modules'][0]['fn'][key].remove(entry)
+                                logs.append(f"The {i+1}'th {key} wire in the top level bf is targeting -1")
+                            except:
+                                entry['tgt'] = 1
+                    except:
+                        continue
             except:
                 continue
 
@@ -85,11 +109,14 @@ def fn_preprocessor(function_network: Dict[str, Any]):
                 except:
                     continue
             else:
-                try:
-                    for (i, entry) in enumerate(fn_ent[key]):
+                try: 
+                    for (i, entry) in enumerate(reversed(fn_ent[key])):
                         if entry['tgt'] == -1:
-                            del fn_ent[key][i]
-                            logs.append(f"The {i + 1}'th {key} wire in the {j + 1}'th fn_array is targeting -1")
+                            try:
+                                fn_ent[key][i].remove(entry)
+                                logs.append(f"The {i+1}'th {key} wire in the {j+1}'th fn_array is targeting -1")
+                            except:
+                                entry['tgt'] = 1
                 except:
                     continue
 

diff --git a/skema/skema-rs/mathml/src/parsers/decapodes_serialization.rs b/skema/skema-rs/mathml/src/parsers/decapodes_serialization.rs
@@ -475,3 +475,49 @@ fn test_serialize_halfar_dome_3_2() {
     let json = to_decapodes_json(wiring_diagram);
     assert_eq!(json, "{\"Var\":[{\"type\":\"infer\",\"name\":\"mult_1\"},{\"type\":\"infer\",\"name\":\"mult_2\"},{\"type\":\"infer\",\"name\":\"•1\"},{\"type\":\"Literal\",\"name\":\"2\"},{\"type\":\"infer\",\"name\":\"sum_1\"},{\"type\":\"infer\",\"name\":\"n\"},{\"type\":\"infer\",\"name\":\"A\"},{\"type\":\"infer\",\"name\":\"•2\"},{\"type\":\"infer\",\"name\":\"mult_3\"},{\"type\":\"infer\",\"name\":\"ρ\"},{\"type\":\"infer\",\"name\":\"g\"},{\"type\":\"infer\",\"name\":\"Γ\"}],\"Op1\":[],\"Op2\":[{\"proj1\":4,\"proj2\":5,\"res\":3,\"op2\":\"/\"},{\"proj1\":3,\"proj2\":7,\"res\":2,\"op2\":\"*\"},{\"proj1\":10,\"proj2\":11,\"res\":9,\"op2\":\"*\"},{\"proj1\":9,\"proj2\":6,\"res\":8,\"op2\":\"^\"},{\"proj1\":2,\"proj2\":8,\"res\":1,\"op2\":\"*\"}],\"Σ\":[{\"sum\":5}],\"Summand\":[{\"summand\":6,\"summation\":1},{\"summand\":4,\"summation\":1}]}");
 }
+
+#[test]
+fn test_serialize_from_image_3_1() {
+    let input = "
+    <math>
+        <mfrac><mrow><mi>∂</mi><mi>H</mi></mrow><mrow><mi>∂</mi><mi>t</mi></mrow></mfrac>
+        <mo>=</mo>
+        <mo>∇</mo>
+        <mo>⋅</mo>
+        <mo>(</mo>
+        <mi>Γ</mi>
+        <msup><mi>H</mi><mrow><mi>n</mi><mo>+</mo><mn>2</mn></mrow></msup>
+        <mo>|</mo><mrow><mo>∇</mo><mi>H</mi></mrow>
+        <msup><mo>|</mo>
+        <mrow><mi>n</mi><mo>−</mo><mn>1</mn></mrow></msup>
+        <mo>∇</mo><mi>H</mi>
+        <mo>)</mo>
+    </math>
+    ";
+    let expression = input.parse::<MathExpressionTree>().unwrap();
+    let wiring_diagram = to_wiring_diagram(&expression);
+    let json = to_decapodes_json(wiring_diagram);
+    assert_eq!(json,"{\"Var\":[{\"type\":\"infer\",\"name\":\"•1\"},{\"type\":\"infer\",\"name\":\"mult_1\"},{\"type\":\"infer\",\"name\":\"mult_2\"},{\"type\":\"infer\",\"name\":\"mult_3\"},{\"type\":\"infer\",\"name\":\"Γ\"},{\"type\":\"infer\",\"name\":\"•2\"},{\"type\":\"infer\",\"name\":\"H\"},{\"type\":\"infer\",\"name\":\"sum_1\"},{\"type\":\"infer\",\"name\":\"n\"},{\"type\":\"Literal\",\"name\":\"2\"},{\"type\":\"infer\",\"name\":\"•3\"},{\"type\":\"infer\",\"name\":\"•4\"},{\"type\":\"infer\",\"name\":\"•5\"},{\"type\":\"infer\",\"name\":\"•6\"},{\"type\":\"Literal\",\"name\":\"1\"},{\"type\":\"infer\",\"name\":\"•7\"},{\"type\":\"infer\",\"name\":\"•8\"}],\"Op1\":[{\"src\":7,\"tgt\":13,\"op1\":\"Grad\"},{\"src\":13,\"tgt\":12,\"op1\":\"Abs\"},{\"src\":7,\"tgt\":16,\"op1\":\"Grad\"},{\"src\":2,\"tgt\":1,\"op1\":\"Div\"},{\"src\":7,\"tgt\":17,\"op1\":\"D(1,t)\"}],\"Op2\":[{\"proj1\":7,\"proj2\":8,\"res\":6,\"op2\":\"^\"},{\"proj1\":5,\"proj2\":6,\"res\":4,\"op2\":\"*\"},{\"proj1\":9,\"proj2\":15,\"res\":14,\"op2\":\"-\"},{\"proj1\":12,\"proj2\":14,\"res\":11,\"op2\":\"^\"},{\"proj1\":4,\"proj2\":11,\"res\":3,\"op2\":\"*\"},{\"proj1\":3,\"proj2\":16,\"res\":2,\"op2\":\"*\"}],\"Σ\":[{\"sum\":8}],\"Summand\":[{\"summand\":9,\"summation\":1},{\"summand\":10,\"summation\":1}]}");
+}
+
+#[test]
+fn test_serialize_from_image_3_2() {
+    let input = "
+    <math>
+        <mi>Γ</mi>
+        <mo>=</mo>
+        <mfrac><mn>2</mn><mrow><mi>n</mi><mo>+</mo><mn>2</mn></mrow></mfrac>
+        <mi>A</mi>
+        <mo>(</mo>
+        <mi>ρ</mi>
+        <mi>g</mi>
+        <msup><mo>)</mo>
+        <mi>n</mi></msup>
+    </math>
+    ";
+    let expression = input.parse::<MathExpressionTree>().unwrap();
+    let s_exp = expression.to_string();
+    let wiring_diagram = to_wiring_diagram(&expression);
+    let json = to_decapodes_json(wiring_diagram);
+    assert_eq!(json, "{\"Var\":[{\"type\":\"infer\",\"name\":\"mult_1\"},{\"type\":\"infer\",\"name\":\"mult_2\"},{\"type\":\"infer\",\"name\":\"•1\"},{\"type\":\"Literal\",\"name\":\"2\"},{\"type\":\"infer\",\"name\":\"sum_1\"},{\"type\":\"infer\",\"name\":\"n\"},{\"type\":\"infer\",\"name\":\"A\"},{\"type\":\"infer\",\"name\":\"•2\"},{\"type\":\"infer\",\"name\":\"mult_3\"},{\"type\":\"infer\",\"name\":\"ρ\"},{\"type\":\"infer\",\"name\":\"g\"},{\"type\":\"infer\",\"name\":\"Γ\"}],\"Op1\":[],\"Op2\":[{\"proj1\":4,\"proj2\":5,\"res\":3,\"op2\":\"/\"},{\"proj1\":3,\"proj2\":7,\"res\":2,\"op2\":\"*\"},{\"proj1\":10,\"proj2\":11,\"res\":9,\"op2\":\"*\"},{\"proj1\":9,\"proj2\":6,\"res\":8,\"op2\":\"^\"},{\"proj1\":2,\"proj2\":8,\"res\":1,\"op2\":\"*\"}],\"Σ\":[{\"sum\":5}],\"Summand\":[{\"summand\":6,\"summation\":1},{\"summand\":4,\"summation\":1}]}");
+}
diff --git a/skema/skema-rs/mathml/src/parsers/generic_mathml.rs b/skema/skema-rs/mathml/src/parsers/generic_mathml.rs
@@ -224,7 +224,7 @@ pub fn mean(input: Span) -> IResult<Operator> {
 }
 
 pub fn grad(input: Span) -> IResult<Operator> {
-    let (s, op) = value(Operator::Grad, alt((ws(tag("𝛁")), ws(tag("&#x2207;")))))(input)?;
+    let (s, op) = value(Operator::Grad, alt((ws(tag("∇")), ws(tag("&#x2207;")))))(input)?;
     Ok((s, op))
 }
 

diff --git a/skema/skema-rs/skema/src/bin/morae.rs b/skema/skema-rs/skema/src/bin/morae.rs
@@ -43,14 +43,14 @@ fn main() {
 
         let host = "localhost";
 
-        //let math_content = module_id2mathml_MET_ast(module_id, host);
+        let math_content = module_id2mathml_MET_ast(module_id, host);
 
         let input_src = "../../data/mml2pn_inputs/testing_eqns/sidarthe_mml.txt";
 
         // This does get a panic with a message, so need to figure out how to forward it
         //let _mathml_ast = get_mathml_asts_from_file(input_src.clone());
 
-        let f = File::open(input_src.clone()).unwrap();
+        /*let f = File::open(input_src.clone()).unwrap();
         let lines = BufReader::new(f).lines();
         let mut deca_vec = Vec::<MathExpressionTree>::new();
         let mut wiring_vec = Vec::<WiringDiagram>::new();
@@ -68,7 +68,7 @@ fn main() {
 
         println!("{:?}", wiring_vec.clone());
         println!("decapode collection: {:?}", decapodescollection.clone());
-
+        */
         let odes = get_FirstOrderODE_vec_from_file(input_src.clone());
 
         //println!("\nmath_content: {:?}", math_content);
@@ -78,7 +78,7 @@ fn main() {
             "\nAMR from mathml: {}\n",
             serde_json::to_string(&PetriNet::from(odes)).unwrap()
         );
-        //println!("\nAMR from code: {:?}", PetriNet::from(math_content));
+        println!("\nAMR from code: {:?}", PetriNet::from(math_content));
     }
     // This is the graph id for the top level function for the core dynamics for our test case.
     else if new_args.arg == *"manual" {