From 7470da50c4984ae93fbbe352f23f06ae2b1ac69a Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Wed, 18 Jan 2023 11:52:13 +0800 Subject: [PATCH 1/3] feat(syntax_version): add "syntax" identifier Signed-off-by: Eval EXEC --- tools/codegen/src/ast/mod.rs | 2 ++ tools/codegen/src/ast/raw/mod.rs | 21 ++++++++++++++++++++ tools/codegen/src/ast/raw/utils.rs | 23 +++++++++++++++++++++- tools/codegen/src/ast/verified/complete.rs | 4 ++++ tools/codegen/src/ast/verified/mod.rs | 3 +++ tools/codegen/src/ast/verified/recover.rs | 3 +++ tools/codegen/src/grammar.pest | 4 ++++ tools/codegen/src/ir/from_ast.rs | 1 + tools/codegen/src/ir/mod.rs | 3 +++ 9 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tools/codegen/src/ast/mod.rs b/tools/codegen/src/ast/mod.rs index 933abd9..982ccb0 100644 --- a/tools/codegen/src/ast/mod.rs +++ b/tools/codegen/src/ast/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod raw; pub(crate) mod verified; +pub use raw::SyntaxVersion; + pub use verified::{ Array, Ast, DefaultContent, DynVec, FieldDecl, FixVec, HasName, ImportStmt, ItemDecl, Option_, Primitive, Struct, Table, TopDecl, Union, UnionItemDecl, diff --git a/tools/codegen/src/ast/raw/mod.rs b/tools/codegen/src/ast/raw/mod.rs index c3d8dac..3444ecd 100644 --- a/tools/codegen/src/ast/raw/mod.rs +++ b/tools/codegen/src/ast/raw/mod.rs @@ -1,16 +1,37 @@ use std::path::PathBuf; +#[cfg(feature = "compiler-plugin")] +use serde::{Deserialize, Serialize}; + use property::Property; mod utils; #[derive(Debug, Default, Property)] pub(crate) struct Ast { + syntax_version: Option, namespace: String, imports: Vec, decls: Vec, } +impl Default for SyntaxVersion { + fn default() -> Self { + Self { version: 1 } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Property)] +#[property(get(public))] +#[cfg_attr( + feature = "compiler-plugin", + derive(Deserialize, Serialize), + serde(deny_unknown_fields) +)] +pub struct SyntaxVersion { + version: usize, +} + #[derive(Debug, Clone, Property)] pub(crate) struct ImportStmt { name: String, diff --git a/tools/codegen/src/ast/raw/utils.rs b/tools/codegen/src/ast/raw/utils.rs index 44db6ab..c2d6d2e 100644 --- a/tools/codegen/src/ast/raw/utils.rs +++ b/tools/codegen/src/ast/raw/utils.rs @@ -4,9 +4,10 @@ use std::{ffi, fs, io::Read as _, path::Path, str::FromStr}; use pest::{error::Error as PestError, iterators::Pairs, Parser as _}; use same_file::is_same_file; -use crate::ast::raw::CustomUnionItemDecl; use crate::{ ast::raw as ast, + ast::raw::CustomUnionItemDecl, + ast::raw::SyntaxVersion, parser, utils::{self, PairsUtils as _}, }; @@ -231,6 +232,22 @@ impl parser::Parser { panic!("grammar should have only one EOI"); } match pair.as_rule() { + parser::Rule::syntax_version_stmt => { + let mut pair = pair.into_inner(); + let syntax_version = SyntaxVersion { + version: pair.next_usize(), + }; + pair.next_should_be_none(); + if ast.syntax_version.is_some() { + // compare ast.syntax_version and syntax_version + // panic if there is a conflict syntax_version + if ast.syntax_version != Some(syntax_version) { + panic!("all schema files' syntax version should be same"); + } + } else { + ast.syntax_version = Some(syntax_version); + } + } parser::Rule::import_stmt => { let mut pair = pair.into_inner(); let node = pair.next_import(path, imported_depth); @@ -312,6 +329,10 @@ impl parser::Parser { if !eoi { panic!("grammar should have only one EOI"); } + + if ast.syntax_version.is_none() { + ast.syntax_version = Some(SyntaxVersion::default()); + } Ok(()) } } diff --git a/tools/codegen/src/ast/verified/complete.rs b/tools/codegen/src/ast/verified/complete.rs index 9970c51..a5d9efe 100644 --- a/tools/codegen/src/ast/verified/complete.rs +++ b/tools/codegen/src/ast/verified/complete.rs @@ -216,7 +216,11 @@ impl super::Ast { let result = decls_result.get(decl.name()).unwrap(); decls.push(Rc::clone(result)); } + + let syntax_version = raw.syntax_version().unwrap().to_owned(); + Self { + syntax_version, namespace, imports, decls, diff --git a/tools/codegen/src/ast/verified/mod.rs b/tools/codegen/src/ast/verified/mod.rs index 6e4e7e0..9052ccd 100644 --- a/tools/codegen/src/ast/verified/mod.rs +++ b/tools/codegen/src/ast/verified/mod.rs @@ -12,11 +12,14 @@ mod recover; pub use default_content::DefaultContent; pub use has_name::HasName; +use crate::ast::SyntaxVersion; + type Deps<'a> = HashMap<&'a str, Rc>; #[derive(Debug, Property)] #[property(get(public))] pub struct Ast { + syntax_version: SyntaxVersion, namespace: String, imports: Vec, decls: Vec>, diff --git a/tools/codegen/src/ast/verified/recover.rs b/tools/codegen/src/ast/verified/recover.rs index 9401774..060f992 100644 --- a/tools/codegen/src/ast/verified/recover.rs +++ b/tools/codegen/src/ast/verified/recover.rs @@ -223,7 +223,10 @@ impl super::Ast { let result = decls_result.get(decl.name()).unwrap(); decls.push(Rc::clone(result)); } + + let syntax_version = ir.syntax_version().to_owned(); Self { + syntax_version, namespace, imports, decls, diff --git a/tools/codegen/src/grammar.pest b/tools/codegen/src/grammar.pest index cecc163..a7c418c 100644 --- a/tools/codegen/src/grammar.pest +++ b/tools/codegen/src/grammar.pest @@ -87,8 +87,12 @@ path_super = @{ "../" } path = { path_super* ~ (identifier ~ "/")* ~ identifier } import_stmt = { "import" ~ (brk)+ ~ path ~ (brk)* ~ stmt_end } +syntax_version = @{ digit+ } +syntax_version_stmt = { "syntax" ~ (brk)* ~ "=" ~ (brk)* ~ syntax_version ~ (brk)* ~ stmt_end} + grammar = { SOI ~ (brk)* ~ + (syntax_version_stmt)? ~ (brk)* ~ (import_stmt ~ (brk)*)* ~ decl_stmt ~ ((brk)* ~ decl_stmt)* ~ (brk)* ~ diff --git a/tools/codegen/src/ir/from_ast.rs b/tools/codegen/src/ir/from_ast.rs index 9e85ee7..8a29327 100644 --- a/tools/codegen/src/ir/from_ast.rs +++ b/tools/codegen/src/ir/from_ast.rs @@ -9,6 +9,7 @@ impl ToIntermediate for ast::Ast { type Ir = super::Ir; fn to_ir(&self) -> Self::Ir { Self::Ir { + syntax_version: self.syntax_version().to_owned(), namespace: self.namespace().to_owned(), imports: self.imports().iter().map(ToIntermediate::to_ir).collect(), decls: self.decls().iter().map(|decl| decl.to_ir()).collect(), diff --git a/tools/codegen/src/ir/mod.rs b/tools/codegen/src/ir/mod.rs index f3a3d36..52f5c23 100644 --- a/tools/codegen/src/ir/mod.rs +++ b/tools/codegen/src/ir/mod.rs @@ -8,10 +8,13 @@ use property::Property; pub use format::Format; pub(crate) use from_ast::ToIntermediate; +use crate::ast::SyntaxVersion; + /// Intermediate file. #[derive(Debug, Property, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct Ir { + syntax_version: SyntaxVersion, namespace: String, imports: Vec, #[serde(rename = "declarations")] From 9d9d1237910f5e1e7f669584cb7d3d0b8f8bb9da Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Wed, 18 Jan 2023 14:37:01 +0800 Subject: [PATCH 2/3] feat(syntax_version): add cargo test for parsing syntax version Signed-off-by: Eval EXEC --- Makefile | 11 ++++- tools/codegen/src/ast/raw/utils.rs | 69 ++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8e39093..ccaa5ab 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ci: @set -eu; \ export RUSTFLAGS='-D warnings'; \ make fmt clippy; \ - make ci-examples ci-crates; \ + make cargo-test ci-examples ci-crates; \ echo "Success!" RUST_DEV_PROJS = examples/ci-tests tests @@ -40,6 +40,15 @@ clippy: cd - > /dev/null; \ done +cargo-test: + @set -eu; \ + for dir in ${RUST_PROJS}; do \ + cd "$${dir}"; \ + cargo test; \ + cd - > /dev/null; \ + done + + ci-msrv: @set -eu; \ for dir in ${RUST_PROD_PROJS}; do \ diff --git a/tools/codegen/src/ast/raw/utils.rs b/tools/codegen/src/ast/raw/utils.rs index c2d6d2e..502794d 100644 --- a/tools/codegen/src/ast/raw/utils.rs +++ b/tools/codegen/src/ast/raw/utils.rs @@ -336,3 +336,72 @@ impl parser::Parser { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::{parser, utils, SyntaxVersion}; + use std::io::Write; + + #[test] + fn test_default_syntax_version_should_be_1_0() { + use utils::ParserUtils; + // get path of file + let mut schema_file = tempfile::NamedTempFile::new().unwrap(); + let _ = schema_file.write(b"array uint32 [byte; 4];").unwrap(); + schema_file.flush().unwrap(); + + let file = schema_file.into_temp_path(); + + let ast = parser::Parser::preprocess(&file).unwrap(); + assert_eq!(ast.syntax_version, Some(SyntaxVersion { version: 1 })); + } + + #[test] + fn test_parse_syntax_version() { + use utils::ParserUtils; + // get path of file + let mut schema_file = tempfile::NamedTempFile::new().unwrap(); + let test_version = SyntaxVersion { version: 7 }; + schema_file + .write_fmt(format_args!("syntax = {};", test_version.version)) + .unwrap(); + let _ = schema_file.write(b"array uint32 [byte; 4];").unwrap(); + schema_file.flush().unwrap(); + + let file = schema_file.into_temp_path(); + + let ast = parser::Parser::preprocess(&file).unwrap(); + assert_eq!(ast.syntax_version, Some(test_version)); + } + + #[test] + #[should_panic] + // if A `syntax = 1` schema file imports a `syntax = 2` schema file, it should panic + fn test_different_syntax_version_should_panic() { + use utils::ParserUtils; + + let mut child_schema_file = tempfile::NamedTempFile::new().unwrap(); + child_schema_file + .write_fmt(format_args!("syntax = 2;")) + .unwrap(); + let _ = child_schema_file.write(b"array uint64 [byte; 8];").unwrap(); + child_schema_file.flush().unwrap(); + + let child_file = child_schema_file.into_temp_path(); + let child_file_path = child_file.to_str().unwrap(); + + let mut root_schema_file = tempfile::NamedTempFile::new().unwrap(); + root_schema_file + .write_fmt(format_args!("syntax = 1;",)) + .unwrap(); + root_schema_file + .write_fmt(format_args!("import {:?}", child_file_path)) + .unwrap(); + let _ = root_schema_file.write(b"array uint32 [byte; 4];").unwrap(); + root_schema_file.flush().unwrap(); + + let file = root_schema_file.into_temp_path(); + + parser::Parser::preprocess(&file).unwrap(); + } +} From aa8d448f248e3972e4bdbffa933c66442df83a3a Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Wed, 22 Mar 2023 13:13:31 +0800 Subject: [PATCH 3/3] feat(syntax_version): codegen/ir: allow deserialize to IR when syntax_version is not present --- tools/codegen/src/ir/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/codegen/src/ir/mod.rs b/tools/codegen/src/ir/mod.rs index 52f5c23..539bf9e 100644 --- a/tools/codegen/src/ir/mod.rs +++ b/tools/codegen/src/ir/mod.rs @@ -14,6 +14,7 @@ use crate::ast::SyntaxVersion; #[derive(Debug, Property, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct Ir { + #[serde(default)] syntax_version: SyntaxVersion, namespace: String, imports: Vec,