Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

More Fn-Preprocessing #612

Merged
merged 4 commits into from
Nov 1, 2023
Merged

More Fn-Preprocessing #612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
211c1bc
added more checks to database.rs and expanded fn-preprocessing
Nov 1, 2023
de63231
hidden dependency for langchain
Nov 1, 2023
0fde253
had wrong version
Nov 1, 2023
77ed7fe
Merge branch 'main' into lieffers/code2amr/fixes4
myedibleenso Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ metal = ["skema[ml]",
"askem_extractions[all]@git+https://github.com/ml4ai/ASKEM-TA1-DataModel"]

# for llm use in skema
llms = ["langchain==0.0.325"]
llms = ["langchain==0.0.325", "openai==0.28.0"]

# dependencies for text reading utilities.
tr = ["skema[ml]", "pyarrow==13.0.0",
Expand Down
45 changes: 36 additions & 9 deletions skema/rest/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,29 @@ def fn_preprocessor(function_network: Dict[str, Any]):
2) metadata being inline for bf entries instead of an index into the metadata_collection -> which we will replace with an index of 2
3) missing function_type field on a bf entry -> will replace with function_type: "IMPORTED"
4) If there is not a body field to a function -> replace "FUNCTION" with "ABSTRACT and set "name":"unknown"
5) NOT DONE YET: In the future we will preprocess about function calls being arguments, in order to simplify extracting the dataflow
5) If there are -1 entries in the metadata for line spans and col spans -> replaced with 1
6) NOT DONE YET: In the future we will preprocess about function calls being arguments, in order to simplify extracting the dataflow
'''

# first we check the top bf level of wires and inline metadata:
keys_to_check = ['bf', 'wff', 'wfopi', 'wfopo', 'wopio']
metadata_keys_to_check = ['line_begin', 'line_end', 'col_begin', 'col_end']
for key in metadata_keys_to_check:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow. That's deep!

try:
for (i, entry) in enumerate(fn_data['modules'][0]['metadata_collection']):
try:
for (j, datum) in enumerate(entry):
try:
if datum[key] == -1:
datum[key] = 1
logs.append(f"The {j + 1}'th metadata in the {i+1} metadata index has -1 for the {key} entry")
except:
continue
except:
continue
except:
continue

for key in keys_to_check:
if key == 'bf':
try:
Expand Down Expand Up @@ -50,10 +68,16 @@ def fn_preprocessor(function_network: Dict[str, Any]):
continue
else:
try:
for (i, entry) in enumerate(fn_data['modules'][0]['fn'][key]):
if entry['tgt'] == -1:
del fn_data['modules'][0]['fn'][key][i]
logs.append(f"The {i + 1}'th {key} wire in the top level bf is targeting -1")
for (i, entry) in enumerate(reversed(fn_data['modules'][0]['fn'][key])):
try:
if entry['tgt'] == -1:
try:
fn_data['modules'][0]['fn'][key].remove(entry)
logs.append(f"The {i+1}'th {key} wire in the top level bf is targeting -1")
except:
entry['tgt'] = 1
except:
continue
except:
continue

Expand Down Expand Up @@ -85,11 +109,14 @@ def fn_preprocessor(function_network: Dict[str, Any]):
except:
continue
else:
try:
for (i, entry) in enumerate(fn_ent[key]):
try:
for (i, entry) in enumerate(reversed(fn_ent[key])):
if entry['tgt'] == -1:
del fn_ent[key][i]
logs.append(f"The {i + 1}'th {key} wire in the {j + 1}'th fn_array is targeting -1")
try:
fn_ent[key][i].remove(entry)
logs.append(f"The {i+1}'th {key} wire in the {j+1}'th fn_array is targeting -1")
except:
entry['tgt'] = 1
except:
continue

Expand Down
8 changes: 4 additions & 4 deletions skema/skema-rs/skema/src/bin/morae.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@ fn main() {

let host = "localhost";

//let math_content = module_id2mathml_MET_ast(module_id, host);
let math_content = module_id2mathml_MET_ast(module_id, host);

let input_src = "../../data/mml2pn_inputs/testing_eqns/sidarthe_mml.txt";

// This does get a panic with a message, so need to figure out how to forward it
//let _mathml_ast = get_mathml_asts_from_file(input_src.clone());

let f = File::open(input_src.clone()).unwrap();
/*let f = File::open(input_src.clone()).unwrap();
let lines = BufReader::new(f).lines();
let mut deca_vec = Vec::<MathExpressionTree>::new();
let mut wiring_vec = Vec::<WiringDiagram>::new();
Expand All @@ -68,7 +68,7 @@ fn main() {

println!("{:?}", wiring_vec.clone());
println!("decapode collection: {:?}", decapodescollection.clone());

*/
let odes = get_FirstOrderODE_vec_from_file(input_src.clone());

//println!("\nmath_content: {:?}", math_content);
Expand All @@ -78,7 +78,7 @@ fn main() {
"\nAMR from mathml: {}\n",
serde_json::to_string(&PetriNet::from(odes)).unwrap()
);
//println!("\nAMR from code: {:?}", PetriNet::from(math_content));
println!("\nAMR from code: {:?}", PetriNet::from(math_content));
}
// This is the graph id for the top level function for the core dynamics for our test case.
else if new_args.arg == *"manual" {
Expand Down
119 changes: 62 additions & 57 deletions skema/skema-rs/skema/src/database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4777,86 +4777,91 @@ pub fn import_wiring(
}
} else {
// this means we are in function scope, concerns on if this is cross attributal or just internal wiring...
let eboxf = gromet.modules[0].attributes[parent_node.contents - 1].clone();
let mut eboxf = gromet.modules[0].r#fn.clone();
if parent_node.contents != 0 {
eboxf = gromet.modules[0].attributes[parent_node.contents - 1].clone();
}

// iterate through all wires of type
for wire in eboxf.wff.unwrap().iter() {
let mut wff_src_tgt: Vec<String> = vec![];
if eboxf.wff.is_some() {
for wire in eboxf.wff.unwrap().iter() {
let mut wff_src_tgt: Vec<String> = vec![];

let src_idx = wire.src; // port index
let src_idx = wire.src; // port index

let src_pif = eboxf.pif.as_ref().unwrap()[(src_idx - 1) as usize].clone(); // src port
let src_pif = eboxf.pif.as_ref().unwrap()[(src_idx - 1) as usize].clone(); // src port

let src_box = src_pif.r#box; // src sub module box number
let src_att = idx; // attribute index of submodule (also opi contents value)
let src_nbox = bf_counter; // nbox value of src opi
let src_box = src_pif.r#box; // src sub module box number
let src_att = idx; // attribute index of submodule (also opi contents value)
let src_nbox = bf_counter; // nbox value of src opi

let tgt_idx = wire.tgt; // port index
let tgt_idx = wire.tgt; // port index

let tgt_pof = eboxf.pof.as_ref().unwrap()[(tgt_idx - 1) as usize].clone(); // tgt port
let tgt_pof = eboxf.pof.as_ref().unwrap()[(tgt_idx - 1) as usize].clone(); // tgt port

let tgt_box = tgt_pof.r#box; // tgt sub module box number
let tgt_att = idx; // attribute index of submodule (also opo contents value)
let tgt_nbox = bf_counter; // nbox value of tgt opo
let tgt_box = tgt_pof.r#box; // tgt sub module box number
let tgt_att = idx; // attribute index of submodule (also opo contents value)
let tgt_nbox = bf_counter; // nbox value of tgt opo

// find the src node
for node in nodes.iter() {
// make sure in correct box
if src_nbox == node.nbox {
// make sure only looking in current attribute nodes for srcs and tgts
if src_att == node.contents {
// matche the box
if (src_box as u32) == node.att_bf_idx as u32 {
// only include nodes with pifs
if node.in_indx.is_some() {
// exclude opo's
if node.n_type != "Opi" {
// iterate through port to check for src
for p in node.in_indx.as_ref().unwrap().iter() {
// push the tgt
if (wire.src as u32) == *p {
wff_src_tgt.push(node.node_id.clone());
// find the src node
for node in nodes.iter() {
// make sure in correct box
if src_nbox == node.nbox {
// make sure only looking in current attribute nodes for srcs and tgts
if src_att == node.contents {
// matche the box
if (src_box as u32) == node.att_bf_idx as u32 {
// only include nodes with pifs
if node.in_indx.is_some() {
// exclude opo's
if node.n_type != "Opi" {
// iterate through port to check for src
for p in node.in_indx.as_ref().unwrap().iter() {
// push the tgt
if (wire.src as u32) == *p {
wff_src_tgt.push(node.node_id.clone());
}
}
}
}
}
}
}
}
}
// finding the tgt node
for node in nodes.iter() {
// make sure in correct box
if tgt_nbox == node.nbox {
// make sure only looking in current attribute nodes for srcs and tgts
if tgt_att == node.contents {
// match internal box
if (tgt_box as u32) == node.att_bf_idx as u32 {
// only include nodes with pofs
if node.out_idx.is_some() {
// exclude opo's
if node.n_type != "Opo" {
// iterate through port to check for tgt
for p in node.out_idx.as_ref().unwrap().iter() {
// push the tgt
if (wire.tgt as u32) == *p {
wff_src_tgt.push(node.node_id.clone());
// finding the tgt node
for node in nodes.iter() {
// make sure in correct box
if tgt_nbox == node.nbox {
// make sure only looking in current attribute nodes for srcs and tgts
if tgt_att == node.contents {
// match internal box
if (tgt_box as u32) == node.att_bf_idx as u32 {
// only include nodes with pofs
if node.out_idx.is_some() {
// exclude opo's
if node.n_type != "Opo" {
// iterate through port to check for tgt
for p in node.out_idx.as_ref().unwrap().iter() {
// push the tgt
if (wire.tgt as u32) == *p {
wff_src_tgt.push(node.node_id.clone());
}
}
}
}
}
}
}
}
}
if wff_src_tgt.len() == 2 {
let e8 = Edge {
src: wff_src_tgt[0].clone(),
tgt: wff_src_tgt[1].clone(),
e_type: String::from("Wire"),
prop: None,
};
edges.push(e8);
if wff_src_tgt.len() == 2 {
let e8 = Edge {
src: wff_src_tgt[0].clone(),
tgt: wff_src_tgt[1].clone(),
e_type: String::from("Wire"),
prop: None,
};
edges.push(e8);
}
}
}
}
Expand Down
Loading