Fn-Preprocessing (#602)

This adds a util function to the REST API that takes in a function
network and checks for all known bugs that can cause a deserialization
error downstream. It then addresses the bugs in a generic fashion to
allow serialization (although the FN is not correct, just digestible).
It also outputs a log which contains every bug it has found and
recording the type of bug and where the bug is inside the FN.

I call this FN-Preprocessing on the FN's generated in the larger
workflows as well.

I also setup some error handling incase the dynamics could not be parsed
in code2amr. Returning a blank/default AMR instead of a crash.

---------

Co-authored-by: Justin <[email protected]>

skema/rest/utils.py

@@ -1,13 +1,101 @@
 from collections import defaultdict
 from typing import Any, Dict
 import itertools as it
-
 from askem_extractions.data_model import AttributeCollection, AttributeType, Mention
 from bs4 import BeautifulSoup, Comment
 
 from skema.rest.schema import TextReadingEvaluationResults, AMRLinkingEvaluationResults
 
 
+def fn_preprocessor(function_network: Dict[str, Any]):
+
+    fn_data = function_network.copy()
+
+    logs = []
+
+    '''
+    We will currently preprocess based on 2 different common bugs
+    1) wire tgt's being -1 -> which we will delete these wires
+    2) metadata being inline for bf entries instead of an index into the metadata_collection -> which we will replace with an index of 2
+    3) missing function_type field on a bf entry -> will replace with function_type: "OTHER"
+    4) If there is not a body field to a function -> replace "FUNCTION" with "ABSTRACT and set "name":"unknown"
+    5) NOT DONE YET: In the future we will preprocess about function calls being arguments, in order to simplify extracting the dataflow 
+    '''
+
+    # first we check the top bf level of wires and inline metadata: 
+    keys_to_check = ['bf', 'wff', 'wfopi', 'wfopo', 'wopio']
+    for key in keys_to_check:
+        if key == 'bf':
+            try:
+                for (i, entry) in enumerate(fn_data['modules'][0]['fn'][key]):
+                    try: 
+                        metadata_obj = entry['metadata']
+                        if not isinstance(metadata_obj, int):
+                            entry['metadata'] = 2
+                            logs.append(f"Inline metadata on {i+1}'th entry in top level bf")
+                    except:
+                        continue
+                    try: 
+                        temp = entry['function_type']
+                    except:
+                        entry['function_type'] = "IMPORTED"
+                        logs.append(f"Missing function_type on {i+1}'th entry in top level bf")
+                    try: 
+                        if entry['function_type'] == "FUNCTION":
+                            temp = entry['body']
+                    except:
+                        entry['function_type'] = "ABSTRACT"
+                        entry['name'] = "Unknown"
+                        logs.append(f"Missing Function body on {i+1}'th entry in top level bf")    
+            except:
+                continue
+        else:
+            try: 
+                for (i, entry) in enumerate(fn_data['modules'][0]['fn'][key]):
+                    if entry['tgt'] == -1:
+                        del fn_data['modules'][0]['fn'][key][i]
+                        logs.append(f"The {i+1}'th {key} wire in the top level bf is targeting -1")
+            except:
+                continue
+
+    # now we iterate through the fn_array and do the same thing
+    for (j,fn_ent) in enumerate(fn_data['modules'][0]['fn_array']):
+        for key in keys_to_check:
+            if key == 'bf':
+                try:
+                    for (i, entry) in enumerate(fn_ent[key]):
+                        try: 
+                            metadata_obj = entry['metadata']
+                            if not isinstance(metadata_obj, int):
+                                entry['metadata'] = 2
+                                logs.append(f"Inline metadata on {i+1}'th bf in the {j+1}'th fn_array")
+                        except:
+                            continue
+                        try: 
+                            temp = entry['function_type']
+                        except:
+                            entry['function_type'] = "IMPORTED"
+                            logs.append(f"Missing function_type on {i+1}'th bf in the {j+1}'th fn_array")
+                        try: 
+                            if entry['function_type'] == "FUNCTION":
+                                temp = entry['body']
+                        except:
+                            entry['function_type'] = "ABSTRACT"
+                            entry['name'] = "Unknown"
+                            logs.append(f"Missing Function body on {i+1}'th bf in the {j+1}'th fn_array")  
+                except:
+                    continue
+            else:
+                try: 
+                    for (i, entry) in enumerate(fn_ent[key]):
+                        if entry['tgt'] == -1:
+                            del fn_ent[key][i]
+                            logs.append(f"The {i+1}'th {key} wire in the {j+1}'th fn_array is targeting -1")
+                except:
+                    continue
+
+    return fn_data, logs
+
 def clean_mml(mml: str) -> str:
     """Cleans/sterilizes pMML for AMR generation service"""
     # FIXME: revisit if JSON deserialization on MORAE side changes

skema/rest/workflows.py

@@ -116,6 +116,7 @@ async def equations_to_amr(data: schema.MmlToAMR):
 @router.post("/code/snippets-to-pn-amr", summary="Code snippets → PetriNet AMR")
 async def code_snippets_to_pn_amr(system: code2fn.System):
     gromet = await code2fn.fn_given_filepaths(system)
+    gromet, logs = utils.fn_preprocessor(gromet)
     res = requests.put(f"{SKEMA_RS_ADDESS}/models/PN", json=gromet)
     if res.status_code != 200:
         return JSONResponse(
@@ -153,6 +154,7 @@ async def code_snippets_to_rn_amr(system: code2fn.System):
 )
 async def repo_to_pn_amr(zip_file: UploadFile = File()):
     gromet = await code2fn.fn_given_filepaths_zip(zip_file)
+    gromet, logs = utils.fn_preprocessor(gromet)
     res = requests.put(f"{SKEMA_RS_ADDESS}/models/PN", json=gromet)
     if res.status_code != 200:
         return JSONResponse(

skema/skema-rs/skema/src/model_extraction.rs

@@ -67,20 +67,42 @@ pub fn get_line_span(
 
 #[allow(non_snake_case)]
 pub fn module_id2mathml_MET_ast(module_id: i64, host: &str) -> Vec<FirstOrderODE> {
+    let mut core_dynamics_ast = Vec::<FirstOrderODE>::new();
+    let mut _metadata_map_ast = HashMap::new();
     let graph = subgraph2petgraph(module_id, host); // makes petgraph of graph
 
     let core_id = find_pn_dynamics(module_id, host); // gives back list of function nodes that might contain the dynamics
-    let _line_span = get_line_span(core_id[0], graph); // get's the line span of function id
+    //let _line_span = get_line_span(core_id[0], graph); // get's the line span of function id
 
     //println!("\n{:?}", line_span);
+    if core_id.len() == 0 {
+        let deriv = Ci {
+            r#type: Some(Function),
+            content: Box::new(MathExpression::Mi(Mi("temp".to_string()))),
+            func_of: None,
+        };
+        let operate = Operator::Subtract;
+        let rhs_arg = MathExpressionTree::Atom(MathExpression::Mi(Mi("temp".to_string())));
+        let rhs = MathExpressionTree::Cons(operate, [rhs_arg].to_vec());
+        let fo_eq = FirstOrderODE {
+            lhs_var: deriv.clone(),
+            func_of: [deriv.clone()].to_vec(), // just place holders for construction
+            with_respect_to: deriv.clone(),    // just place holders for construction
+            rhs: rhs,
+        };
+        core_dynamics_ast.push(fo_eq);
+    } else {
+        (core_dynamics_ast, _metadata_map_ast) =
+        subgrapg2_core_dyn_MET_ast(core_id[0], host).unwrap();
+    }
 
     //println!("function_core_id: {:?}", core_id[0].clone());
     //println!("module_id: {:?}\n", module_id.clone());
     // 4.5 now to check if of those expressions, if they are arithmetric in nature
 
     // 5. pass id to subgrapg2_core_dyn to get core dynamics
-    let (core_dynamics_ast, _metadata_map_ast) =
-        subgrapg2_core_dyn_MET_ast(core_id[0], host).unwrap();
+    //let (core_dynamics_ast, _metadata_map_ast) =
+        //subgrapg2_core_dyn_MET_ast(core_id[0], host).unwrap();
 
     core_dynamics_ast
 }