refactor: refactor parser. 1. remove pkg in module 2. Parser traversa…

…l import files from dfs to bfs

Signed-off-by: he1pa <[email protected]>

kclvm/api/src/capi_test.rs

@@ -1,5 +1,6 @@
 use crate::service::capi::*;
 use crate::{call, gpyrpc::*};
+use kclvm_utils::path::PathPrefix;
 use once_cell::sync::Lazy;
 use prost::Message;
 use serde::de::DeserializeOwned;
@@ -9,7 +10,6 @@ use std::fs;
 use std::os::raw::c_char;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
-
 const TEST_DATA_PATH: &str = "./src/testdata";
 static TEST_MUTEX: Lazy<Mutex<i32>> = Lazy::new(|| Mutex::new(0i32));
 
@@ -101,8 +101,17 @@ fn test_c_api_get_schema_type_mapping() {
         "get-schema-type-mapping.json",
         "get-schema-type-mapping.response.json",
         |r| {
+            let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
             for (_, s_ty) in &mut r.schema_type_mapping {
-                s_ty.filename = s_ty.filename.replace('/', "").replace('\\', "")
+                let filename = {
+                    let filename = s_ty.filename.adjust_canonicalization();
+                    match filename.strip_prefix(root.to_str().unwrap()) {
+                        Some(f) => f.to_string(),
+                        None => s_ty.filename.clone(),
+                    }
+                };
+
+                s_ty.filename = filename.replace('.', "").replace('/', "").replace('\\', "")
             }
         },
     );

kclvm/api/src/service/service_impl.rs

@@ -188,7 +188,7 @@ impl KclvmServiceImpl {
             deps: result
                 .deps
                 .iter()
-                .map(|p| p.to_str().unwrap().to_string())
+                .map(|p| p.path.to_str().unwrap().to_string())
                 .collect(),
             errors: result.errors.into_iter().map(|e| e.into_error()).collect(),
         })

kclvm/ast/src/ast.rs

@@ -422,7 +422,6 @@ impl Program {
 #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
 pub struct Module {
     pub filename: String,
-    pub pkg: String,
     pub doc: Option<NodeRef<String>>,
     pub name: String,
     pub body: Vec<NodeRef<Stmt>>,

kclvm/ast/src/tests.rs

@@ -179,7 +179,6 @@ fn test_try_from_for_nameconstant() {
 fn test_filter_schema_with_no_schema() {
     let ast_mod = Module {
         filename: "".to_string(),
-        pkg: "".to_string(),
         doc: Some(node_ref!("".to_string())),
         name: "".to_string(),
         body: vec![],
@@ -193,7 +192,6 @@ fn test_filter_schema_with_no_schema() {
 fn test_filter_schema_with_one_schema() {
     let mut ast_mod = Module {
         filename: "".to_string(),
-        pkg: "".to_string(),
         doc: Some(node_ref!("".to_string())),
         name: "".to_string(),
         body: vec![],
@@ -210,7 +208,6 @@ fn test_filter_schema_with_one_schema() {
 fn test_filter_schema_with_mult_schema() {
     let mut ast_mod = Module {
         filename: "".to_string(),
-        pkg: "".to_string(),
         doc: Some(node_ref!("".to_string())),
         name: "".to_string(),
         body: vec![],

kclvm/loader/Cargo.toml

@@ -15,6 +15,7 @@ kclvm-parser = {path = "../parser"}
 kclvm-sema = {path = "../sema"}
 kclvm-error = {path = "../error"}
 kclvm-query = {path = "../query"}
+kclvm-utils = {path = "../utils"}
 maplit = "1.0.2"
 
 [dev-dependencies]

kclvm/loader/src/lib.rs

@@ -23,6 +23,7 @@ use kclvm_sema::{
     },
     ty::{Type, TypeRef},
 };
+use kclvm_utils::path::PathPrefix;
 use std::path::PathBuf;
 
 type Errors = IndexSet<Diagnostic>;
@@ -195,8 +196,9 @@ pub fn load_packages_with_cache(
     for path in &packages.paths {
         let path_str = path
             .to_str()
-            .ok_or(anyhow::anyhow!("path {} to str failed", path.display()))?;
-        if let Some(files) = gs.get_sema_db().get_file_sema(path_str) {
+            .ok_or(anyhow::anyhow!("path {} to str failed", path.display()))?
+            .adjust_canonicalization();
+        if let Some(files) = gs.get_sema_db().get_file_sema(&path_str) {
             for symbol_ref in files.get_symbols() {
                 if let Some(symbol) = symbols.get_symbol(*symbol_ref) {
                     let def_ty = match symbol.get_definition() {

kclvm/parser/src/file_graph.rs

@@ -1,23 +1,49 @@
+use std::{collections::HashMap, path::PathBuf};
+
 use indexmap::IndexMap;
-use petgraph::visit::EdgeRef;
-use std::path::{Path, PathBuf};
+use kclvm_ast::ast::Module;
+use petgraph::{prelude::StableDiGraph, visit::EdgeRef};
+use std::hash::Hash;
+/// File with package info
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub struct PkgFile {
+    pub path: PathBuf,
+    pub pkg_path: String,
+}
+
+impl PkgFile {
+    pub fn canonicalize(&self) -> PathBuf {
+        match self.path.canonicalize() {
+            Ok(p) => p.clone(),
+            _ => self.path.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub struct Pkg {
+    pub pkg_name: String,
+    pub pkg_root: String,
+}
+
+pub type PkgMap = HashMap<PkgFile, Pkg>;
 
 /// A graph of files, where each file depends on zero or more other files.
 #[derive(Default)]
-pub struct FileGraph {
-    graph: petgraph::stable_graph::StableDiGraph<PathBuf, ()>,
-    path_to_node_index: IndexMap<PathBuf, petgraph::graph::NodeIndex>,
+pub struct PkgFileGraph {
+    graph: StableDiGraph<PkgFile, ()>,
+    path_to_node_index: IndexMap<PkgFile, petgraph::graph::NodeIndex>,
 }
 
-impl FileGraph {
+impl PkgFileGraph {
     /// Sets a file to depend on the given other files.
     ///
     /// For example, if the current graph has file A depending on B, and
     /// `update_file(pathA, &[pathC])` was called, then this function will remove the edge
     /// from A to B, and add an edge from A to C.
-    pub fn update_file<'a, I: IntoIterator<Item = &'a PathBuf>>(
+    pub fn update_file<'a, I: IntoIterator<Item = &'a PkgFile>>(
         &mut self,
-        from_path: &Path,
+        from_path: &PkgFile,
         to_paths: I,
     ) {
         let from_node_index = self.get_or_insert_node_index(from_path);
@@ -39,68 +65,152 @@ impl FileGraph {
     }
 
     /// Returns true if the given file is in the graph
-    pub fn contains_file(&mut self, path: &Path) -> bool {
-        self.path_to_node_index.contains_key(path)
+    pub fn contains_file(&self, file: &PkgFile) -> bool {
+        contains_file(file, &self.path_to_node_index)
     }
 
     /// Returns a list of the direct dependencies of the given file.
     /// (does not include all transitive dependencies)
     /// The file path must be relative to the root of the file graph.
-    pub fn dependencies_of(&self, path: &Path) -> Vec<&PathBuf> {
-        let node_index = self
-            .path_to_node_index
-            .get(path)
-            .expect("path not in graph");
-        self.graph
-            .edges(*node_index)
-            .map(|edge| &self.graph[edge.target()])
-            .collect::<Vec<_>>()
+    pub fn dependencies_of(&self, file: &PkgFile) -> Vec<PkgFile> {
+        dependencies_of(file, &self.graph, &self.path_to_node_index)
     }
 
-    /// Returns a list of files in the order they should be compiled
-    /// Or a list of files that are part of a cycle, if one exists
-    pub fn toposort(&self) -> Result<Vec<PathBuf>, Vec<PathBuf>> {
-        match petgraph::algo::toposort(&self.graph, None) {
-            Ok(indices) => Ok(indices
-                .into_iter()
-                .rev()
-                .map(|n| self.graph[n].clone())
-                .collect::<Vec<_>>()),
-            Err(err) => {
-                // toposort function in the `petgraph` library doesn't return the cycle itself,
-                // so we need to use Tarjan's algorithm to find one instead
-                let strongly_connected_components = petgraph::algo::tarjan_scc(&self.graph);
-
-                // a strongly connected component is a cycle if it has more than one node
-                // let's just return the first one we find
-                let cycle = match strongly_connected_components
-                    .into_iter()
-                    .find(|component| component.len() > 1)
-                {
-                    Some(vars) => vars,
-                    None => vec![err.node_id()],
-                };
-                Err(cycle
-                    .iter()
-                    .map(|n| self.graph[*n].clone())
-                    .collect::<Vec<_>>())
-            }
-        }
+    pub fn toposort(&self) -> Result<Vec<PkgFile>, Vec<PkgFile>> {
+        toposort(&self.graph)
     }
 
     /// Returns all paths.
     #[inline]
-    pub fn paths(&self) -> Vec<PathBuf> {
+    pub fn paths(&self) -> Vec<PkgFile> {
         self.path_to_node_index.keys().cloned().collect::<Vec<_>>()
     }
 
-    fn get_or_insert_node_index(&mut self, path: &Path) -> petgraph::graph::NodeIndex {
-        if let Some(node_index) = self.path_to_node_index.get(path) {
+    fn get_or_insert_node_index(&mut self, file: &PkgFile) -> petgraph::graph::NodeIndex {
+        if let Some(node_index) = self.path_to_node_index.get(file) {
             return *node_index;
         }
 
-        let node_index = self.graph.add_node(path.to_owned());
-        self.path_to_node_index.insert(path.to_owned(), node_index);
+        let node_index = self.graph.add_node(file.to_owned());
+        self.path_to_node_index.insert(file.to_owned(), node_index);
         node_index
     }
+
+    pub fn file_path_graph(
+        &self,
+    ) -> (
+        StableDiGraph<PathBuf, ()>,
+        IndexMap<PathBuf, petgraph::prelude::NodeIndex>,
+    ) {
+        let mut graph = StableDiGraph::new();
+        let mut node_map = IndexMap::new();
+        for node in self.graph.node_indices() {
+            let path = self.graph[node].path.clone();
+            let idx = graph.add_node(path.clone());
+            node_map.insert(path, idx);
+        }
+        for edge in self.graph.edge_indices() {
+            if let Some((source, target)) = self.graph.edge_endpoints(edge) {
+                let source_path = self.graph[source].path.clone();
+                let target_path = self.graph[target].path.clone();
+                graph.add_edge(
+                    node_map.get(&source_path).unwrap().clone(),
+                    node_map.get(&target_path).unwrap().clone(),
+                    (),
+                );
+            }
+        }
+        (graph, node_map)
+    }
+
+    pub fn pkg_graph(
+        &self,
+        pkgs: &HashMap<String, Vec<Module>>,
+    ) -> (
+        StableDiGraph<String, ()>,
+        IndexMap<String, petgraph::prelude::NodeIndex>,
+    ) {
+        let mut graph = StableDiGraph::new();
+        let mut node_map = IndexMap::new();
+
+        for pkg in pkgs.keys() {
+            let idx = graph.add_node(pkg.clone());
+            node_map.insert(pkg.clone(), idx);
+        }
+
+        for node in self.graph.node_indices() {
+            let path = self.graph[node].pkg_path.clone();
+            let idx = graph.add_node(path.clone());
+            node_map.insert(path, idx);
+        }
+        for edge in self.graph.edge_indices() {
+            if let Some((source, target)) = self.graph.edge_endpoints(edge) {
+                let source_path = self.graph[source].pkg_path.clone();
+                let target_path = self.graph[target].pkg_path.clone();
+                graph.add_edge(
+                    node_map.get(&source_path).unwrap().clone(),
+                    node_map.get(&target_path).unwrap().clone(),
+                    (),
+                );
+            }
+        }
+        (graph, node_map)
+    }
+}
+
+/// Returns a list of files in the order they should be compiled
+/// Or a list of files that are part of a cycle, if one exists
+pub fn toposort<T>(graph: &StableDiGraph<T, ()>) -> Result<Vec<T>, Vec<T>>
+where
+    T: Clone,
+{
+    match petgraph::algo::toposort(graph, None) {
+        Ok(indices) => Ok(indices
+            .into_iter()
+            .rev()
+            .map(|n| graph[n].clone())
+            .collect::<Vec<_>>()),
+        Err(err) => {
+            // toposort function in the `petgraph` library doesn't return the cycle itself,
+            // so we need to use Tarjan's algorithm to find one instead
+            let strongly_connected_components = petgraph::algo::tarjan_scc(&graph);
+            // a strongly connected component is a cycle if it has more than one node
+            // let's just return the first one we find
+            let cycle = match strongly_connected_components
+                .into_iter()
+                .find(|component| component.len() > 1)
+            {
+                Some(vars) => vars,
+                None => vec![err.node_id()],
+            };
+            Err(cycle.iter().map(|n| graph[*n].clone()).collect::<Vec<_>>())
+        }
+    }
+}
+
+/// Returns a list of the direct dependencies of the given file.
+/// (does not include all transitive dependencies)
+/// The file path must be relative to the root of the file graph.
+pub fn dependencies_of<T>(
+    node: &T,
+    graph: &StableDiGraph<T, ()>,
+    id_map: &IndexMap<T, petgraph::prelude::NodeIndex>,
+) -> Vec<T>
+where
+    T: Clone + Hash + Eq + PartialEq,
+{
+    let node_index = id_map.get(node).expect("node not in graph");
+    graph
+        .edges(*node_index)
+        .map(|edge| &graph[edge.target()])
+        .map(|node| node.clone())
+        .collect::<Vec<_>>()
+}
+
+/// Returns true if the given file is in the graph
+pub fn contains_file<T>(node: &T, id_map: &IndexMap<T, petgraph::prelude::NodeIndex>) -> bool
+where
+    T: Clone + Hash + Eq + PartialEq,
+{
+    id_map.contains_key(node)
 }