From 4c586614a5d0d241f5301b78031ce217963e295b Mon Sep 17 00:00:00 2001 From: David Teller Date: Wed, 10 Oct 2018 10:16:48 +0200 Subject: [PATCH] Resolves #176 - Getting rid of TokenWriter::{list, tagged_tuple, Tree} and the companion code in ast.rs This changeset simplifies trait `io::TokenWriter`, getting rid of some baggage that makes work on entropy needlessly complicated. For backwards compatibility reason, we keep a `io::deprecated::TokenWriterWithTree`, which is essentially a copy of the old `io::TokenWriter`, along with a `io::deprecated::TokenWriterTreeAdapter`, which converts from the old form to the new one. This change should make future work on entropy a bit simpler, improve performance and this has already helped locate a few bugs in the generic implementation of encoding. This change will need to be followed up by a corresponding work on the TokenReader, once we have a clear idea of exactly what needs to be done. 1. `TokenWriter` doesn't define a type `Tree` anymore. Rather, on success, all methods return `()`. Consequently, all implementations of `Serializer`, which returned `TokenWriter::Tree` on success now return `()` on success. This includes both hardcoded implementations and implementations extracted from the webidl. 2. `TokenWriter` doesn't take a `Vec` of children for lists and tagged tuples anymore. Consequently, we adapt the code generator to not create these `Vec` of children anymore. 3. Similarly, the generic implementation of encoding (which might not be useful anymore, but it's still here for the moment) needs to be adapted to the new `TokenWriter`. This requires a few contorsions in binjs_generic/src/io/encode.rs. 4. Both methods `Encoder::encode`, which give access to all token writers now use `TokenWriterTreeAdapter` for older implementations of `TokenWriterWithThree`. 5. We adapt the entropy dictionary builder to this new `TokenWriter`, which essentially means removing now useless code. 6. Other implementations of `TokenWriter` move to `TokenWriterWithTree`, which changes essentially the interface name. 7. Adding methods `from_rc_string` to all types derived from `SharedString`. 8. Adapting tests to the API changes. --- crates/binjs_es6/src/io.rs | 54 +++---- crates/binjs_generate_library/src/lib.rs | 54 +++---- crates/binjs_generic/src/io/encode.rs | 190 +++++++++++++++------- crates/binjs_generic/src/io/mod.rs | 7 +- crates/binjs_io/src/entropy/model.rs | 11 +- crates/binjs_io/src/io/deprecated.rs | 196 +++++++++++++++++++++++ crates/binjs_io/src/io/mod.rs | 91 +++-------- crates/binjs_io/src/multipart/mod.rs | 10 +- crates/binjs_io/src/multipart/write.rs | 4 +- crates/binjs_io/src/simple.rs | 10 +- crates/binjs_io/src/xml.rs | 6 +- crates/binjs_shared/src/shared_string.rs | 5 +- examples/compare_compression.rs | 2 +- tests/test_paths.rs | 19 ++- tests/test_roundtrip.rs | 6 +- tests/test_simple.rs | 2 +- 16 files changed, 444 insertions(+), 223 deletions(-) create mode 100644 crates/binjs_io/src/io/deprecated.rs diff --git a/crates/binjs_es6/src/io.rs b/crates/binjs_es6/src/io.rs index 26d0ba207..cfee71f6a 100644 --- a/crates/binjs_es6/src/io.rs +++ b/crates/binjs_es6/src/io.rs @@ -1,4 +1,4 @@ -use binjs_io::{ self, Deserialization, Guard, TokenReader, TokenReaderError, TokenWriterError }; +use binjs_io::{ self, Deserialization, Guard, TokenReader, TokenReaderError, TokenWriterTreeAdapter, TokenWriterError }; pub use binjs_io::{ Serialization, TokenSerializer, TokenWriter }; use binjs_shared::{ FieldName, IdentifierName, InterfaceName, Offset, PropertyKey, SharedString, self }; @@ -131,7 +131,7 @@ impl Serializer where W: TokenWriter { writer } } - pub fn serialize(&mut self, value: T, path: &mut IOPath) -> Result where Self: Serialization { + pub fn serialize(&mut self, value: T, path: &mut IOPath) -> Result<(), TokenWriterError> where Self: Serialization { (self as &mut Serialization).serialize(value, path) } } @@ -143,99 +143,99 @@ impl TokenSerializer for Serializer where W: TokenWriter { } impl Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.bool_at(value, path) } } impl Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: bool, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: bool, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.bool_at(Some(value), path) } } impl Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.float_at(value, path) } } impl Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: f64, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: f64, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.float_at(Some(value), path) } } impl Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: u32, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: u32, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.unsigned_long_at(value, path) } } impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.bool_at(value.clone(), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a bool, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a bool, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.bool_at(Some(*value), path) } } impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.float_at(value.clone(), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a f64, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a f64, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.float_at(Some(*value), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a u32, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a u32, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.unsigned_long_at(value.clone(), path) } } /* impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: Option<&'a str>, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: Option<&'a str>, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.string_at(value, path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a str, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a str, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.string_at(Some(value), path) } } */ impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a SharedString, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a SharedString, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.string_at(Some(value), path) } } impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.string_at(value.as_ref(), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a IdentifierName, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a IdentifierName, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.identifier_name_at(Some(&value), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a PropertyKey, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a PropertyKey, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.property_key_at(Some(&value), path) } } impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.identifier_name_at(value.as_ref(), path) } } impl<'a, W> Serialization> for Serializer where W: TokenWriter { - fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result { + fn serialize(&mut self, value: &'a Option, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.property_key_at(value.as_ref(), path) } } impl<'a, W> Serialization for Serializer where W: TokenWriter { - fn serialize(&mut self, _: &'a Offset, path: &mut IOPath) -> Result { + fn serialize(&mut self, _: &'a Offset, path: &mut IOPath) -> Result<(), TokenWriterError> { self.writer.offset_at(path) } } @@ -275,9 +275,9 @@ impl Encoder { } pub fn encode<'a, AST>(&self, format: &'a mut binjs_io::Format, ast: &'a AST) -> Result>, TokenWriterError> where - Serializer : Serialization, - Serializer : Serialization, - Serializer : Serialization, + Serializer> : Serialization, &'a AST>, + Serializer> : Serialization, &'a AST>, + Serializer> : Serialization, &'a AST>, // Serializer> : Serialization, &'a AST> /* #[cfg(multistream)] @@ -290,14 +290,14 @@ impl Encoder { match *format { binjs_io::Format::Simple { .. } => { let writer = binjs_io::simple::TreeTokenWriter::new(); - let mut serializer = Serializer::new(writer); + let mut serializer = Serializer::new(TokenWriterTreeAdapter::new(writer)); serializer.serialize(ast, &mut path)?; let (data, _) = serializer.done()?; Ok(Box::new(data)) } binjs_io::Format::Multipart { ref mut targets, .. } => { let writer = binjs_io::multipart::TreeTokenWriter::new(targets.clone()); - let mut serializer = Serializer::new(writer); + let mut serializer = Serializer::new(TokenWriterTreeAdapter::new(writer)); serializer.serialize(ast, &mut path)?; let (data, _) = serializer.done()?; Ok(Box::new(data)) @@ -323,7 +323,7 @@ impl Encoder { } binjs_io::Format::XML => { let writer = binjs_io::xml::Encoder::new(); - let mut serializer = Serializer::new(writer); + let mut serializer = Serializer::new(TokenWriterTreeAdapter::new(writer)); serializer.serialize(ast, &mut path)?; let (data, _) = serializer.done()?; Ok(Box::new(data)) diff --git a/crates/binjs_generate_library/src/lib.rs b/crates/binjs_generate_library/src/lib.rs index a19f5a47b..fc60039ef 100644 --- a/crates/binjs_generate_library/src/lib.rs +++ b/crates/binjs_generate_library/src/lib.rs @@ -275,7 +275,7 @@ impl FromJSON for {name} {{ let to_writer = format!(" impl<'a, W> Serialization for Serializer where W: TokenWriter {{ - fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing string enum {name}\"); let str = match *value {{ {variants} @@ -574,22 +574,21 @@ impl ToJSON for {name} {{ let to_writer = format!(" impl<'a, W> Serialization> for Serializer where W: TokenWriter {{ - fn serialize(&mut self, value: &'a Option<{name}>, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, value: &'a Option<{name}>, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing optional sum {name}\"); match *value {{ None => {{ let interface_name = InterfaceName::from_str(\"{null}\"); - self.writer.enter_tagged_tuple_at(&interface_name, 0, path)?; - let result = self.writer.tagged_tuple_at(&interface_name, &[], path)?; - self.writer.exit_tagged_tuple_at(&interface_name, path)?; - Ok(result) + self.writer.enter_tagged_tuple_at(&interface_name, &[], path)?; + self.writer.exit_tagged_tuple_at(&interface_name, &[], path)?; + Ok(()) }} Some(ref sum) => (self as &mut Serialization).serialize(sum, path) }} }} }} impl<'a, W> Serialization for Serializer where W: TokenWriter {{ - fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing sum {name}\"); match *value {{ {variants} @@ -841,17 +840,15 @@ impl<'a> Walker<'a> for ViewMut{name}<'a> {{ impl<'a, W> Serialization for Serializer where W: TokenWriter {{ - fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, value: &'a {name}, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing list {name}\"); self.writer.enter_list_at(value.len(), path)?; - let mut children = Vec::with_capacity(value.len()); for child in value {{ // All the children of the list share the same path. - children.push(self.serialize(child, path)?); + self.serialize(child, path)?; }} - let result = self.writer.list_at(children, path)?; self.writer.exit_list_at(path)?; - Ok(result) + Ok(()) }} }} ", @@ -1059,43 +1056,44 @@ impl Deserialization> for Deserializer where R: TokenRea .len(); let to_writer = format!(" impl<'a, W> Serialization> for Serializer where W: TokenWriter {{ - fn serialize(&mut self, value: &'a Option<{name}>, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, value: &'a Option<{name}>, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing optional tagged tuple {name}\"); match *value {{ None => {{ let interface_name = InterfaceName::from_str(\"{null}\"); - self.writer.enter_tagged_tuple_at(&interface_name, 0, path)?; - let result = self.writer.tagged_tuple_at(&interface_name, &[], path)?; - self.writer.exit_tagged_tuple_at(&interface_name, path)?; - Ok(result) + self.writer.enter_tagged_tuple_at(&interface_name, &[], path)?; + self.writer.exit_tagged_tuple_at(&interface_name, &[], path)?; + Ok(()) }} Some(ref sum) => (self as &mut Serialization).serialize(sum, path) }} }} }} impl<'a, W> Serialization for Serializer where W: TokenWriter {{ - fn serialize(&mut self, {value}: &'a {name}, path: &mut IOPath) -> Result {{ + fn serialize(&mut self, {value}: &'a {name}, path: &mut IOPath) -> Result<(), TokenWriterError> {{ debug!(target: \"serialize_es6\", \"Serializing tagged tuple {name}\"); let interface_name = InterfaceName::from_str(\"{name}\"); // String is shared + let field_names = [{field_names}]; - self.writer.enter_tagged_tuple_at(&interface_name, {len}, path)?; + self.writer.enter_tagged_tuple_at(&interface_name, &field_names, path)?; path.enter_interface(interface_name.clone()); - let {mut} children = Vec::with_capacity({len}); {fields} - let result = self.writer.{tagged_tuple}(&interface_name, &children, path); path.exit_interface(interface_name.clone()); - self.writer.exit_tagged_tuple_at(&interface_name, path)?; + self.writer.exit_tagged_tuple_at(&interface_name, &field_names, path)?; - result + Ok(()) }} }} ", - mut = if len > 0 { "mut" } else { "" }, value = if len > 0 { "value" } else { "_" }, null = null_name, name = name, - len = len, - tagged_tuple = if interface.is_scope() { "tagged_scoped_tuple_at" } else { "tagged_tuple_at" }, + field_names = interface.contents() + .fields() + .iter() + .map(|field| format!("&FieldName::from_str(\"{field_name}\")", + field_name = field.name().to_str())) + .format(", "), fields = interface.contents() .fields() .iter() @@ -1105,9 +1103,9 @@ impl<'a, W> Serialization for Serializer where W: TokenWriter let field_name = FieldName::from_str(\"{field_name}\"); let path_item = ({index}, field_name.clone()); // String is shared path.enter_field(path_item.clone()); - let child = (self as &mut Serialization).serialize(&value.{rust_field_name}, path); + let result = (self as &mut Serialization).serialize(&value.{rust_field_name}, path); path.exit_field(path_item); - children.push((field_name, child?));", + result?;", index = index, field_name = field.name().to_str(), rust_field_name = field.name().to_rust_identifier_case())) diff --git a/crates/binjs_generic/src/io/encode.rs b/crates/binjs_generic/src/io/encode.rs index 99925b873..f5bf43768 100644 --- a/crates/binjs_generic/src/io/encode.rs +++ b/crates/binjs_generic/src/io/encode.rs @@ -2,7 +2,7 @@ use util::type_of; use binjs_io::{ TokenWriter, TokenWriterError }; use binjs_meta::spec::*; -use binjs_shared::{ FieldName, InterfaceName, SharedString }; +use binjs_shared::{ FieldName, InterfaceName, SharedString, self }; use std; use std::cell::*; @@ -10,6 +10,8 @@ use std::cell::*; use json::JsonValue as JSON; use json::object::Object as Object; +type Path = binjs_shared::ast::Path; + #[derive(Debug)] pub enum Error { Mismatch { expected: String, got: String }, @@ -45,11 +47,11 @@ pub trait Encode { fn done(self) -> Result<(Self::Data, Self::Statistics), std::io::Error>; } -pub struct Encoder<'a, B, Tree> where B: TokenWriter { +pub struct Encoder<'a, B> where B: TokenWriter { grammar: &'a Spec, builder: RefCell, } -impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { +impl<'a, B> Encoder<'a, B> where B: TokenWriter { pub fn new(syntax: &'a Spec, builder: B) -> Self { Encoder { grammar: syntax, @@ -57,14 +59,15 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { } } - /// Encode a JSON into a SerializeTree based on a grammar. + /// Encode a JSON into a Serialize() based on a grammar. /// This step doesn't perform any interesting check on the JSON. - pub fn encode(&self, value: &JSON) -> Result { + pub fn encode(&self, value: &JSON) -> Result<(), Error> { let root = self.grammar.get_root(); let node = self.grammar.get_root_name(); - self.encode_from_named_type(value, &root, &node, false) + let mut path = Path::new(); + self.encode_from_named_type(&mut path, value, &root, &node, false) } - pub fn encode_from_named_type(&self, value: &JSON, named: &NamedType, node: &NodeName, is_optional: bool) -> Result { + pub fn encode_from_named_type(&self, path: &mut Path, value: &JSON, named: &NamedType, node: &NodeName, is_optional: bool) -> Result<(), Error> { match *named { NamedType::StringEnum(ref enum_) => { let string = value.as_str() @@ -79,7 +82,7 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { { let string = SharedString::from_string(string.to_string()); return self.builder.borrow_mut() - .string(Some(&string)) + .string_at(Some(&string), path) .map_err(Error::TokenWriterError) } Err(Error::NoSuchLiteral { @@ -87,31 +90,28 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { }) } NamedType::Typedef(ref type_) => - return self.encode_from_type(value, type_, node, is_optional), + return self.encode_from_type(path, value, type_, node, is_optional), NamedType::Interface(ref interface) => { debug!(target:"encode", "Attempting to encode value {:?} with interface {:?}", value, interface.name()); + let interface_name = InterfaceName::from_rc_string(interface.name() + .to_rc_string() + .clone()); + path.enter_interface(interface_name.clone()); match *value { JSON::Object(ref object) => { debug!(target:"encode", "Attempting to encode object {:?} with interface {:?}", value["type"].as_str(), interface.name()); - let interface_name = interface.name() - .to_string(); let type_field = object.get("type") .ok_or_else(|| Error::missing_field("type", node))? .as_str() .ok_or_else(|| Error::missing_field("type", node))?; if interface_name == type_field { - let fields = interface.contents().fields(); - let contents = self.encode_structure(object, fields, interface.name())?; // Write the contents with the tag of the refined interface. - let interface_name = InterfaceName::from_string(type_field.to_string()); - let labelled = self.builder - .borrow_mut() - .tagged_tuple(&interface_name, &contents) - .map_err(Error::TokenWriterError)?; // FIXME: We should consolidate spec::InterfaceName and shared::InterfaceName - return Ok(labelled) + let fields = interface.contents() + .fields(); + self.encode_structure(path, &interface_name, object, fields, interface.name())?; } else { return Err(Error::Mismatch { - expected: interface_name.clone(), + expected: interface_name.as_str().to_string(), got: type_field.to_string() }) } @@ -119,11 +119,19 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { JSON::Null if is_optional => { // Write the contents with the tag of the refined interface. let null_name = InterfaceName::from_string(self.grammar.get_null_name().to_str().to_string()); - let labelled = self.builder - .borrow_mut() - .tagged_tuple(&null_name, &[]) - .map_err(Error::TokenWriterError)?; - return Ok(labelled) + { + self.builder + .borrow_mut() + .enter_tagged_tuple_at(&null_name, &[], path) + .map_err(Error::TokenWriterError)?; + } + // No fields. + { + self.builder + .borrow_mut() + .exit_tagged_tuple_at(&null_name, &[], path) + .map_err(Error::TokenWriterError)?; + } } _ => { return Err(Error::Mismatch { @@ -132,52 +140,96 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { }) } } + path.exit_interface(interface_name); + Ok(()) } } } - pub fn encode_from_type(&self, value: &JSON, type_: &Type, node: &NodeName, is_optional: bool) -> Result { + pub fn encode_from_type(&self, path: &mut Path, value: &JSON, type_: &Type, node: &NodeName, is_optional: bool) -> Result<(), Error> { debug!(target:"encode", "value {:?} within node {:?}, type {:?}", value, node, type_); - self.encode_from_type_spec(value, type_.spec(), node, is_optional || type_.is_optional()) + self.encode_from_type_spec(path, value, type_.spec(), node, is_optional || type_.is_optional()) } - fn encode_from_type_spec(&self, value: &JSON, spec: &TypeSpec, node: &NodeName, is_optional: bool) -> Result { + fn encode_from_type_spec(&self, path: &mut Path, value: &JSON, spec: &TypeSpec, node: &NodeName, is_optional: bool) -> Result<(), Error> { use binjs_meta::spec::TypeSpec::*; match (spec, value) { (&Array { contents: ref kind, .. }, &JSON::Array(ref array)) => { - let mut encoded = Vec::new(); + { + self.builder.borrow_mut() + .enter_list_at(array.len(), path) + .map_err(Error::TokenWriterError)?; + } for item in array { - let item = self.encode_from_type(item, kind, node, false)?; - encoded.push(item); + self.encode_from_type(path, item, kind, node, false)?; + } + { + self.builder.borrow_mut() + .exit_list_at(path) + .map_err(Error::TokenWriterError)?; } - let result = self.builder.borrow_mut().list(encoded) - .map_err(Error::TokenWriterError)?; - return Ok(result) } (&Boolean, &JSON::Boolean(b)) => { - return self.builder.borrow_mut().bool(Some(b)) + return self.builder + .borrow_mut() + .bool_at(Some(b), path) + .map_err(Error::TokenWriterError) + } + (&String, _) if value.is_string() => { + let string = value.as_str() + .unwrap() + .to_string(); // Checked just above. + return self.builder.borrow_mut() + .string_at(Some(&SharedString::from_string(string)), path) .map_err(Error::TokenWriterError) } - (&String, _) | (&IdentifierName, _) | (&PropertyKey, _) if value.is_string() => { + (&String, &JSON::Null) => { + return self.builder + .borrow_mut() + .string_at(None, path) + .map_err(Error::TokenWriterError) + } + (&IdentifierName, _) if value.is_string() => { + let string = value.as_str() + .unwrap() + .to_string(); // Checked just above. + return self.builder.borrow_mut() + .identifier_name_at(Some(&binjs_shared::IdentifierName::from_string(string)), path) + .map_err(Error::TokenWriterError) + } + (&IdentifierName, &JSON::Null) => { + return self.builder + .borrow_mut() + .identifier_name_at(None, path) + .map_err(Error::TokenWriterError) + } + (&PropertyKey, _) if value.is_string() => { let string = value.as_str() .unwrap() .to_string(); // Checked just above. return self.builder.borrow_mut() - .string(Some(&SharedString::from_string(string))) + .property_key_at(Some(&binjs_shared::PropertyKey::from_string(string)), path) .map_err(Error::TokenWriterError) } - (&String, &JSON::Null) | (&IdentifierName, &JSON::Null) | (&PropertyKey, &JSON::Null) if is_optional => { - return self.builder.borrow_mut().string(None) + (&PropertyKey, &JSON::Null) => { + return self.builder + .borrow_mut() + .property_key_at(None, path) .map_err(Error::TokenWriterError) } (&Number, &JSON::Number(x)) => - return self.builder.borrow_mut().float(Some(x.into())) + return self.builder + .borrow_mut() + .float_at(Some(x.into()), path) .map_err(Error::TokenWriterError), (&UnsignedLong, &JSON::Number(x)) => - return self.builder.borrow_mut().unsigned_long(x.into()) + return self.builder + .borrow_mut() + .unsigned_long_at(x.into(), path) .map_err(Error::TokenWriterError), (&NamedType(ref name), _) => { - let named_type = self.grammar.get_type_by_name(name) + let named_type = self.grammar + .get_type_by_name(name) .ok_or_else(|| Error::NoSuchType(name.to_string().clone()))?; - return self.encode_from_named_type(value, &named_type, node, is_optional) + return self.encode_from_named_type(path, value, &named_type, node, is_optional) } (&TypeSum(ref sum), &JSON::Object(ref object)) => { // Sums may only be used to encode objects or null. @@ -189,14 +241,14 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { .ok_or_else(|| Error::NoSuchInterface(name.to_string().clone()))?; let named_type = self.grammar.get_type_by_name(name) .ok_or_else(|| Error::NoSuchInterface(name.to_string().clone()))?; - return self.encode_from_named_type(value, &named_type, node, is_optional) + return self.encode_from_named_type(path, value, &named_type, node, is_optional) } (&TypeSum(_), &JSON::Null) if is_optional => { // The `Null` interface can be substituted. let null_name = self.grammar.get_null_name(); let null_type = self.grammar.get_type_by_name(self.grammar.get_null_name()) .unwrap_or_else(|| panic!("Could not find type named {}", null_name.to_str())); - return self.encode_from_named_type(value, &null_type, null_name, /* is_optional = */ true) + return self.encode_from_named_type(path, value, &null_type, null_name, /* is_optional = */ true) } _ => {} } @@ -205,31 +257,57 @@ impl<'a, B, Tree> Encoder<'a, B, Tree> where B: TokenWriter { got: value.dump() }) } - fn encode_structure(&self, object: &Object, fields: &[Field], node: &NodeName) -> Result, Error> { - let mut result = Vec::with_capacity(fields.len()); - 'fields: for field in fields { - let field_name = FieldName::from_string(field.name().to_str().to_string()); + fn encode_structure(&self, path: &mut Path, interface_name: &InterfaceName, object: &Object, fields: &[Field], node: &NodeName) -> Result<(), Error> { + // FIXME: Once we unify FieldName between binjs_meta and everything else, + // we should be able to avoid all these costly conversions. Not a priority + // atm, as binjs_generic is used only for testing purposes. + let field_names : Vec<_> = fields.iter() + .map(|field| FieldName::from_rc_string(field + .name() + .to_rc_string() + .clone())) + .collect(); + let field_names_refs : Vec<_> = field_names.iter() + .collect(); + { + self.builder + .borrow_mut() + .enter_tagged_tuple_at(&interface_name, field_names_refs.as_ref(), path)?; + } + + 'fields: for (i, (field, field_name)) in fields + .iter() + .zip(field_names.iter()) + .enumerate() + { + path.enter_field((i, field_name.clone())); if let Some(source) = object.get(field.name().to_string()) { - let encoded = self.encode_from_type(source, field.type_(), node, false)?; - result.push((field_name, encoded)) + self.encode_from_type(path, source, field.type_(), node, false)?; } else if field.type_().is_optional() { - let encoded = self.encode_from_type(&JSON::Null, field.type_(), node, false)?; - result.push((field_name, encoded)) + self.encode_from_type(path, &JSON::Null, field.type_(), node, false)?; } else { debug!("Error in {:?}", object); return Err(Error::missing_field(field.name().to_string(), node)); } + path.exit_field((i, field_name.clone())); + } + + { + self.builder + .borrow_mut() + .exit_tagged_tuple_at(&interface_name, field_names_refs.as_ref(), path)?; } - Ok(result) + + Ok(()) } } -impl<'a, B, Tree> Encode for Encoder<'a, B, Tree> where B: TokenWriter { +impl<'a, B> Encode for Encoder<'a, B> where B: TokenWriter { type Data = B::Data; type Statistics = B::Statistics; fn encode(&self, value: &JSON) -> Result<(), std::io::Error> { - (self as &Encoder<'a, B, Tree>).encode(value) + (self as &Encoder<'a, B>).encode(value) .map(|_| ()) .map_err(|err| { std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{:?}", err)) diff --git a/crates/binjs_generic/src/io/mod.rs b/crates/binjs_generic/src/io/mod.rs index fe8df4a73..afd316626 100644 --- a/crates/binjs_generic/src/io/mod.rs +++ b/crates/binjs_generic/src/io/mod.rs @@ -45,16 +45,17 @@ impl Encoder { } pub fn encode(&self, grammar: &binjs_meta::spec::Spec, format: &mut binjs_io::Format, ast: &JSON) -> Result>, std::io::Error> { + use binjs_io::TokenWriterTreeAdapter; match *format { binjs_io::Format::Simple { .. } => { - let writer = binjs_io::simple::TreeTokenWriter::new(); + let writer = TokenWriterTreeAdapter::new(binjs_io::simple::TreeTokenWriter::new()); let mut encoder = encode::Encoder::new(grammar, writer); encoder.generic_encode(ast)?; let (data, _) = encoder.done()?; Ok(Box::new(data)) } binjs_io::Format::Multipart { ref mut targets, .. } => { - let writer = binjs_io::multipart::TreeTokenWriter::new(targets.clone()); + let writer = TokenWriterTreeAdapter::new(binjs_io::multipart::TreeTokenWriter::new(targets.clone())); let mut encoder = encode::Encoder::new(grammar, writer); encoder.generic_encode(ast)?; let (data, _) = encoder.done()?; @@ -80,7 +81,7 @@ impl Encoder { Ok(Box::new(data)) } binjs_io::Format::XML => { - let writer = binjs_io::xml::Encoder::new(); + let writer = TokenWriterTreeAdapter::new(binjs_io::xml::Encoder::new()); let mut encoder = encode::Encoder::new(grammar, writer); encoder.generic_encode(ast)?; let (data, _) = encoder.done()?; diff --git a/crates/binjs_io/src/entropy/model.rs b/crates/binjs_io/src/entropy/model.rs index 7754fbb97..c56c4644d 100644 --- a/crates/binjs_io/src/entropy/model.rs +++ b/crates/binjs_io/src/entropy/model.rs @@ -274,7 +274,6 @@ impl<'a> DictionaryBuilder<'a> { } impl<'a> TokenWriter for DictionaryBuilder<'a> { - type Tree = (); type Statistics = usize /* placeholder */; type Data = [u8;0]; @@ -329,19 +328,13 @@ impl<'a> TokenWriter for DictionaryBuilder<'a> { Ok(()) } - fn enter_tagged_tuple_at(&mut self, tag: &InterfaceName, _children: usize, path: &IOPath) -> Result<(), TokenWriterError> { + fn enter_tagged_tuple_at(&mut self, tag: &InterfaceName, _children: &[&FieldName], path: &IOPath) -> Result<(), TokenWriterError> { Self::add_instance_to_path(self.max_path_depth, tag.clone(), path, &mut self.dictionary.interface_name_by_path); Self::add_instance_to_strings(tag.clone(), &mut self.instances_of_strings_in_current_file.interface_name_instances); Ok(()) } - fn tagged_tuple(&mut self, _tag: &InterfaceName, _children: &[(FieldName, Self::Tree)]) -> Result<(), TokenWriterError> { - // Needed to keep the driver happy. - Ok(()) - } - - fn list_at(&mut self, _children: Vec, _path: &IOPath) -> Result<(), TokenWriterError> { - // Needed to keep the driver happy. + fn offset_at(&mut self, _path: &IOPath) -> Result<(), TokenWriterError> { Ok(()) } } diff --git a/crates/binjs_io/src/io/deprecated.rs b/crates/binjs_io/src/io/deprecated.rs new file mode 100644 index 000000000..6a75f8862 --- /dev/null +++ b/crates/binjs_io/src/io/deprecated.rs @@ -0,0 +1,196 @@ +//! The original version of trait `io::TokenWriter` included some baggage that proves unnecessary for most +//! encodings, and slowed down encoding. This file provides `TokenWriterWithTree`, which is essentially a +//! copy of the old `io::TokenWriter` along with a `TokenWriterTreeAdapter` which wraps a `TokenWriterWithTree` +//! as a new `io::TokenWriter`. + +use binjs_shared::{ IdentifierName, InterfaceName, FieldName, PropertyKey, SharedString }; + +use ::{ Path, TokenWriter, TokenWriterError }; + +use std::fmt::Display; + +// Deprecated trait. Any new implementation should use TokenWriter. +pub trait TokenWriterWithTree where Self::Statistics: Display + Sized + std::ops::Add + Default { + /// Statistics produced by this writer. + type Statistics; + + type Tree; + + /// The type of data generated by this writer. + /// Typically some variant of `Vec`. + type Data: AsRef<[u8]>; + + /// Finish writing, produce data. + fn done(self) -> Result<(Self::Data, Self::Statistics), TokenWriterError>; + + /// Write a tagged tuple. + /// + /// The number of items is specified by the grammar, so it MAY not be + /// recorded by the `TokenWriter`. + /// + /// By convention, a null tagged tuple is the special tagged tuple "null", + /// with no children. + fn tagged_tuple(&mut self, _tag: &InterfaceName, _children: &[(&FieldName, Self::Tree)]) -> Result; + + /// Write a list. + /// + /// By opposition to a tuple, the number of items is variable and MUST + /// be somehow recorded by the `TokenWriter`. + fn list(&mut self, Vec) -> Result; + + /// Write a single UTF-8 string. + /// + /// If specified, the string MUST be UTF-8. + fn string(&mut self, _: Option<&SharedString>) -> Result { + unimplemented!() + } + + /// Write a single UTF-8 value from a string enumeration. + /// + /// The default implementation uses `self.string``, but some encodings may use + /// the extra information e.g. to represent the enumeration by an index in the + /// list of possible values, or to encode string enums as interfaces. + fn string_enum(&mut self, str: &SharedString) -> Result { + self.string(Some(str)) + } + + /// Write a single number. + fn float(&mut self, _: Option) -> Result { + unimplemented!() + } + + /// Write a single u32. + fn unsigned_long(&mut self, _: u32) -> Result { + unimplemented!() + } + + /// Write single bool. + // FIXME: Split `bool` from `maybe_bool`. + fn bool(&mut self, _: Option) -> Result { + unimplemented!() + } + + /// Write the number of bytes left in this tuple. + fn offset(&mut self) -> Result { + unimplemented!() + } + + fn property_key(&mut self, value: Option<&PropertyKey>) -> Result { + let string = value.map(PropertyKey::as_shared_string); + self.string(string) + } + + fn identifier_name(&mut self, value: Option<&IdentifierName>) -> Result { + let string = value.map(IdentifierName::as_shared_string); + self.string(string) + } +} + +pub struct TokenWriterTreeAdapter where T: TokenWriterWithTree { + writer: T, + + /// Invariant: This stack is never empty. + stack: Vec>, +} +impl TokenWriterTreeAdapter where T: TokenWriterWithTree { + pub fn new(writer: T) -> Self { + TokenWriterTreeAdapter { + writer, + stack: vec![vec![]], + } + } + + pub fn top_mut(&mut self) -> &mut Vec { + self.stack.last_mut() + .expect("Empty stack while replacing last child") + } +} + +impl TokenWriter for TokenWriterTreeAdapter where T: TokenWriterWithTree { + type Statistics = T::Statistics; + type Data = T::Data; + + fn done(mut self) -> Result<(Self::Data, Self::Statistics), TokenWriterError> { + assert_eq!(self.stack.len(), 1, "The stack started with 1 item (the toplevel), it must end with 1 item."); + self.stack.pop(); + self.writer.done() + } + + fn enter_tagged_tuple_at(&mut self, _tag: &InterfaceName, _children: &[&FieldName], _path: &Path) -> Result<(), TokenWriterError> { + self.stack.push(vec![]); + Ok(()) + } + + fn exit_tagged_tuple_at(&mut self, tag: &InterfaceName, children: &[&FieldName], _path: &Path) -> Result<(), TokenWriterError> { + let values = self.stack.pop() // We pushed this `vec` in the call to `enter_tagged_tuple_at`. + .expect("Empty stack while popping"); + let children : Vec<_> = children.iter() + .cloned() + .zip(values.into_iter()) + .collect(); + let child = self.writer.tagged_tuple(tag, children.as_ref())?; + self.top_mut() + .push(child); + Ok(()) + } + + fn enter_list_at(&mut self, _len: usize, _path: &Path) -> Result<(), TokenWriterError> { + self.stack.push(vec![]); + Ok(()) + } + fn exit_list_at(&mut self, _path: &Path) -> Result<(), TokenWriterError> { + let values = self.stack.pop() // We pushed this `vec` in the call to `enter_list_at`. + .expect("Empty stack while popping"); + let child = self.writer.list(values)?; + self.top_mut() + .push(child); + Ok(()) + } + + fn string_at(&mut self, value: Option<&SharedString>, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.string(value)?; + self.top_mut() + .push(child); + Ok(()) + } + + fn string_enum_at(&mut self, value: &SharedString, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.string_enum(value)?; + self.top_mut() + .push(child); + Ok(()) + } + + fn float_at(&mut self, value: Option, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.float(value)?; + self.top_mut() + .push(child); + Ok(()) + } + fn unsigned_long_at(&mut self, value: u32, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.unsigned_long(value)?; + self.top_mut() + .push(child); + Ok(()) + } + fn bool_at(&mut self, value: Option, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.bool(value)?; + self.top_mut() + .push(child); + Ok(()) + } + fn offset_at(&mut self, _path: &Path) -> Result<(), TokenWriterError> { + let child = self.writer.offset()?; + self.top_mut() + .push(child); + Ok(()) + } + fn property_key_at(&mut self, value: Option<&PropertyKey>, path: &Path) -> Result<(), TokenWriterError> { + let string = value.map(PropertyKey::as_shared_string); + self.string_at(string, path) + } + fn identifier_name_at(&mut self, value: Option<&IdentifierName>, path: &Path) -> Result<(), TokenWriterError> { + let string = value.map(IdentifierName::as_shared_string); + self.string_at(string, path) + } +} diff --git a/crates/binjs_io/src/io/mod.rs b/crates/binjs_io/src/io/mod.rs index 45fc17d59..a97a18b60 100644 --- a/crates/binjs_io/src/io/mod.rs +++ b/crates/binjs_io/src/io/mod.rs @@ -23,6 +23,11 @@ use std::rc::Rc; #[cfg(multistream)] use itertools::Itertools; +mod deprecated; + +pub use self::deprecated::{ TokenWriterWithTree, TokenWriterTreeAdapter }; + + /// An API for printing the binary representation and its structural /// interpretation of the file. /// @@ -324,9 +329,6 @@ pub trait TokenReader: FileStructurePrinter where Self::Error: Debug + From<::To /// may be used both for debugging purposes and for encodings that depend /// on the current position in the AST (e.g. entropy coding). pub trait TokenWriter where Self::Statistics: Display + Sized + Add + Default { - /// The type of trees manipulated by this writer. - type Tree; - /// Statistics produced by this writer. type Statistics; @@ -344,16 +346,8 @@ pub trait TokenWriter where Self::Statistics: Display + Sized + Add + Default { /// /// By convention, a null tagged tuple is the special tagged tuple "null", /// with no children. - fn tagged_tuple(&mut self, _tag: &InterfaceName, _children: &[(FieldName, Self::Tree)]) -> Result { - unimplemented!() - } - fn tagged_tuple_at(&mut self, tag: &InterfaceName, children: &[(FieldName, Self::Tree)], _path: &Path) -> Result { - self.tagged_tuple(tag, children) - } - fn enter_tagged_tuple_at(&mut self, _tag: &InterfaceName, _children: usize, _path: &Path) -> Result<(), TokenWriterError> { - Ok(()) - } - fn exit_tagged_tuple_at(&mut self, _tag: &InterfaceName, _path: &Path) -> Result<(), TokenWriterError> { + fn enter_tagged_tuple_at(&mut self, _tag: &InterfaceName, _children: &[&FieldName], _path: &Path) -> Result<(), TokenWriterError>; + fn exit_tagged_tuple_at(&mut self, _tag: &InterfaceName, _children: &[&FieldName], _path: &Path) -> Result<(), TokenWriterError> { Ok(()) } @@ -361,15 +355,7 @@ pub trait TokenWriter where Self::Statistics: Display + Sized + Add + Default { /// /// By opposition to a tuple, the number of items is variable and MUST /// be somehow recorded by the `TokenWriter`. - fn list(&mut self, Vec) -> Result { - unimplemented!() - } - fn list_at(&mut self, items: Vec, _path: &Path) -> Result { - self.list(items) - } - fn enter_list_at(&mut self, _len: usize, _path: &Path) -> Result<(), TokenWriterError> { - Ok(()) - } + fn enter_list_at(&mut self, _len: usize, _path: &Path) -> Result<(), TokenWriterError>; fn exit_list_at(&mut self, _path: &Path) -> Result<(), TokenWriterError> { Ok(()) } @@ -377,75 +363,36 @@ pub trait TokenWriter where Self::Statistics: Display + Sized + Add + Default { /// Write a single UTF-8 string. /// /// If specified, the string MUST be UTF-8. - fn string(&mut self, Option<&SharedString>) -> Result { - unimplemented!() - } - fn string_at(&mut self, value: Option<&SharedString>, _path: &Path) -> Result { - self.string(value) - } + fn string_at(&mut self, value: Option<&SharedString>, _path: &Path) -> Result<(), TokenWriterError>; /// Write a single UTF-8 value from a string enumeration. /// /// The default implementation uses `self.string``, but some encodings may use /// the extra information e.g. to represent the enumeration by an index in the /// list of possible values, or to encode string enums as interfaces. - fn string_enum(&mut self, str: &SharedString) -> Result { - self.string(Some(str)) - } - fn string_enum_at(&mut self, value: &SharedString, path: &Path) -> Result { - self.string_at(Some(value), path) - } + fn string_enum_at(&mut self, value: &SharedString, path: &Path) -> Result<(), TokenWriterError>; /// Write a single number. - fn float(&mut self, Option) -> Result { - unimplemented!() - } - fn float_at(&mut self, value: Option, _path: &Path) -> Result { - self.float(value) - } + fn float_at(&mut self, value: Option, _path: &Path) -> Result<(), TokenWriterError>; /// Write a single u32. - fn unsigned_long(&mut self, u32) -> Result { - unimplemented!() - } - fn unsigned_long_at(&mut self, value: u32, _path: &Path) -> Result { - self.unsigned_long(value) - } + fn unsigned_long_at(&mut self, value: u32, _path: &Path) -> Result<(), TokenWriterError>; /// Write single bool. - // FIXME: Split `bool` from `maybe_bool`. - fn bool(&mut self, Option) -> Result { - unimplemented!() - } - fn bool_at(&mut self, value: Option, _path: &Path) -> Result { - self.bool(value) - } + fn bool_at(&mut self, value: Option, _path: &Path) -> Result<(), TokenWriterError>; /// Write the number of bytes left in this tuple. - fn offset(&mut self) -> Result { - unimplemented!() - } - fn offset_at(&mut self, _path: &Path) -> Result { - self.offset() - } + fn offset_at(&mut self, _path: &Path) -> Result<(), TokenWriterError>; - fn property_key(&mut self, value: Option<&PropertyKey>) -> Result { + fn property_key_at(&mut self, value: Option<&PropertyKey>, path: &Path) -> Result<(), TokenWriterError> { let string = value.map(PropertyKey::as_shared_string); - self.string(string) - } - fn property_key_at(&mut self, value: Option<&PropertyKey>, _path: &Path) -> Result { - self.property_key(value) + self.string_at(string, path) } - // FIXME: Split `property_key` from `maybe_property_key`. - fn identifier_name(&mut self, value: Option<&IdentifierName>) -> Result { + fn identifier_name_at(&mut self, value: Option<&IdentifierName>, path: &Path) -> Result<(), TokenWriterError> { let string = value.map(IdentifierName::as_shared_string); - self.string(string) - } - fn identifier_name_at(&mut self, value: Option<&IdentifierName>, _path: &Path) -> Result { - self.identifier_name(value) + self.string_at(string, path) } - // FIXME: Split `identifier_name` from `maybe_identifier_name`. } @@ -509,7 +456,7 @@ pub trait Serialization where W: TokenWriter, T: Sized { /// Serialize a piece of data. /// /// `path` indicates the path from the root of the AST. - fn serialize(&mut self, data: T, path: &mut Path) -> Result; + fn serialize(&mut self, data: T, path: &mut Path) -> Result<(), TokenWriterError>; } pub trait TokenSerializer where W: TokenWriter { fn done(self) -> Result<(W::Data, W::Statistics), TokenWriterError>; diff --git a/crates/binjs_io/src/multipart/mod.rs b/crates/binjs_io/src/multipart/mod.rs index d90592ef9..2509f8e4f 100644 --- a/crates/binjs_io/src/multipart/mod.rs +++ b/crates/binjs_io/src/multipart/mod.rs @@ -194,7 +194,7 @@ fn test_multipart_io() { use binjs_shared::{ FieldName, InterfaceName, SharedString }; use ::CompressionTarget; - use io::{ Guard, TokenReader, TokenWriter }; + use io::{ Guard, TokenReader, TokenWriterWithTree }; use multipart::*; use std::fs::*; @@ -226,7 +226,7 @@ fn test_multipart_io() { { options.reset(); - let mut writer : ::multipart::TreeTokenWriter = TreeTokenWriter::new(options.clone()); + let mut writer = TreeTokenWriter::new(options.clone()); writer.string(Some(&SharedString::from_str("simple string"))) .expect("Writing simple string"); @@ -274,9 +274,9 @@ fn test_multipart_io() { let item_1 = writer.string(Some(&SharedString::from_str("bar"))).unwrap(); let item_2 = writer.float(Some(3.1415)).unwrap(); writer.tagged_tuple(&InterfaceName::from_str("some tuple"), &[ - (FieldName::from_str("abc"), item_0), - (FieldName::from_str("def"), item_1), - (FieldName::from_str("value"), item_2) + (&FieldName::from_str("abc"), item_0), + (&FieldName::from_str("def"), item_1), + (&FieldName::from_str("value"), item_2) ]) .expect("Writing trivial tagged tuple"); diff --git a/crates/binjs_io/src/multipart/write.rs b/crates/binjs_io/src/multipart/write.rs index 426aa8ac5..da0babd5c 100644 --- a/crates/binjs_io/src/multipart/write.rs +++ b/crates/binjs_io/src/multipart/write.rs @@ -666,7 +666,7 @@ impl TreeTokenWriter { } } -impl TokenWriter for TreeTokenWriter { +impl TokenWriterWithTree for TreeTokenWriter { type Tree = Tree; type Data = Box<[u8]>; type Statistics = Statistics; @@ -761,7 +761,7 @@ impl TokenWriter for TreeTokenWriter { // - index in the grammar table (varnum); // - for each item, in the order specified // - the item (see item) - fn tagged_tuple(&mut self, name: &InterfaceName, children: &[(FieldName, Self::Tree)]) -> Result { + fn tagged_tuple(&mut self, name: &InterfaceName, children: &[(&FieldName, Self::Tree)]) -> Result { let data; let description = NodeDescription { kind: name.clone() diff --git a/crates/binjs_io/src/simple.rs b/crates/binjs_io/src/simple.rs index 0dcfe6885..e42a382d1 100644 --- a/crates/binjs_io/src/simple.rs +++ b/crates/binjs_io/src/simple.rs @@ -444,7 +444,7 @@ impl TreeTokenWriter { Ok(self.register(result)) } } -impl TokenWriter for TreeTokenWriter { +impl TokenWriterWithTree for TreeTokenWriter { type Tree = AbstractTree; type Data = Vec; type Statistics = Statistics; @@ -559,7 +559,7 @@ impl TokenWriter for TreeTokenWriter { /// - field names (string, \0 terminated) /// - /// - contents - fn tagged_tuple(&mut self, tag: &InterfaceName, children: &[(FieldName, Self::Tree)]) -> Result { + fn tagged_tuple(&mut self, tag: &InterfaceName, children: &[(&FieldName, Self::Tree)]) -> Result { debug!(target: "simple_writer", "TreeTokenWriter: tagged_tuple"); let mut prefix = Vec::new(); prefix.extend_from_str(""); @@ -620,7 +620,7 @@ impl ::FormatProvider for FormatProvider { #[test] fn test_simple_io() { use binjs_shared::{ FieldName, InterfaceName, SharedString }; - + use io::TokenWriterWithTree; use std::fs::*; use std::io::{ Cursor, Write }; @@ -717,8 +717,8 @@ fn test_simple_io() { let item_1 = writer.float(Some(3.1415)).unwrap(); writer.tagged_tuple(&InterfaceName::from_str("BindingIdentifier"), &[ - (FieldName::from_str("label"), item_0), - (FieldName::from_str("value"), item_1) + (&FieldName::from_str("label"), item_0), + (&FieldName::from_str("value"), item_1) ]) .expect("Writing trivial tagged tuple"); diff --git a/crates/binjs_io/src/xml.rs b/crates/binjs_io/src/xml.rs index 452126ab2..95bbb00eb 100644 --- a/crates/binjs_io/src/xml.rs +++ b/crates/binjs_io/src/xml.rs @@ -2,7 +2,7 @@ //! //! Used mainly to extract statistics and/or to compare with XML-based compression mechanisms. -use ::{ TokenWriter, TokenWriterError }; +use ::{ TokenWriterWithTree, TokenWriterError }; use binjs_shared::{ FieldName, InterfaceName, SharedString }; @@ -80,7 +80,7 @@ impl Encoder { Ok(result) } } -impl TokenWriter for Encoder { +impl TokenWriterWithTree for Encoder { type Statistics = u32; // Ignored for the time being. type Tree = Rc; type Data = Vec; @@ -101,7 +101,7 @@ impl TokenWriter for Encoder { self.register(SubTree::String(data.map(Clone::clone))) } - fn tagged_tuple(&mut self, tag: &InterfaceName, items: &[(FieldName, Self::Tree)]) -> Result { + fn tagged_tuple(&mut self, tag: &InterfaceName, items: &[(&FieldName, Self::Tree)]) -> Result { self.register(SubTree::Node { name: tag.as_shared_string() .clone(), diff --git a/crates/binjs_shared/src/shared_string.rs b/crates/binjs_shared/src/shared_string.rs index ff2adefef..b02b05413 100644 --- a/crates/binjs_shared/src/shared_string.rs +++ b/crates/binjs_shared/src/shared_string.rs @@ -64,7 +64,7 @@ impl SharedString { pub fn from_str(value: &'static str) -> Self { SharedString::Static(value) } - pub fn from_rc(value: Rc) -> Self { + pub fn from_rc_string(value: Rc) -> Self { SharedString::Dynamic(value) } pub fn from_string(value: String) -> Self { @@ -84,6 +84,9 @@ macro_rules! shared_string { pub fn from_string(value: String) -> Self { $name(shared_string::SharedString::from_string(value)) } + pub fn from_rc_string(value: std::rc::Rc) -> Self { + $name(shared_string::SharedString::from_rc_string(value)) + } pub fn as_str(&self) -> &str { self.0.as_str() } diff --git a/examples/compare_compression.rs b/examples/compare_compression.rs index 617ebd28d..7f6908bf1 100644 --- a/examples/compare_compression.rs +++ b/examples/compare_compression.rs @@ -135,7 +135,7 @@ fn main() { let data: Box> = match format { Format::Multipart { ref mut targets, .. } => { targets.reset(); - let writer = binjs::io::multipart::TreeTokenWriter::new(targets.clone()); + let writer = binjs::io::TokenWriterTreeAdapter::new(binjs::io::multipart::TreeTokenWriter::new(targets.clone())); let mut serializer = binjs::specialized::es6::io::Serializer::new(writer); serializer.serialize(&ast, &mut IOPath::new()) .expect("Could not encode AST"); diff --git a/tests/test_paths.rs b/tests/test_paths.rs index 6b5e8664e..8e64b8d42 100644 --- a/tests/test_paths.rs +++ b/tests/test_paths.rs @@ -44,6 +44,7 @@ enum Event { TaggedTupleAt { interface: InterfaceName, path: IOPath, + len: usize, }, ListAt { len: usize, @@ -64,7 +65,6 @@ struct PathTraceWriter { } impl TokenWriter for PathTraceWriter { - type Tree = (); type Statistics = usize; type Data = [u8;0]; fn identifier_name_at(&mut self, value: Option<&IdentifierName>, path: &IOPath) -> Result<(), TokenWriterError> { @@ -102,34 +102,39 @@ impl TokenWriter for PathTraceWriter { }); Ok(()) } - fn tagged_tuple_at(&mut self, tag: &InterfaceName, _children: &[(FieldName, Self::Tree)], path: &IOPath) -> Result { + fn enter_tagged_tuple_at(&mut self, tag: &InterfaceName, children: &[&FieldName], path: &IOPath) -> Result<(), TokenWriterError> { self.trace.borrow_mut().push(Event::TaggedTupleAt { interface: tag.clone(), - path: path.clone() + path: path.clone(), + len: children.len(), }); Ok(()) } - fn list_at(&mut self, items: Vec<()>, path: &IOPath) -> Result { + fn enter_list_at(&mut self, len: usize, path: &IOPath) -> Result<(), TokenWriterError> { self.trace.borrow_mut().push(Event::ListAt { - len: items.len(), + len, path: path.clone() }); Ok(()) } - fn float_at(&mut self, value: Option, path: &IOPath) -> Result { + fn float_at(&mut self, value: Option, path: &IOPath) -> Result<(), TokenWriterError> { self.trace.borrow_mut().push(Event::FloatAt { value: value.clone(), path: path.clone() }); Ok(()) } - fn property_key_at(&mut self, value: Option<&PropertyKey>, path: &IOPath) -> Result { + fn property_key_at(&mut self, value: Option<&PropertyKey>, path: &IOPath) -> Result<(), TokenWriterError> { self.trace.borrow_mut().push(Event::PropertyKeyAt { value: value.cloned(), path: path.clone() }); Ok(()) } + fn offset_at(&mut self, _path: &IOPath) -> Result<(), TokenWriterError> { + // Nothing to do. + Ok(()) + } fn done(self) -> Result<(Self::Data, Self::Statistics), TokenWriterError> { unimplemented!() } diff --git a/tests/test_roundtrip.rs b/tests/test_roundtrip.rs index 23bdbe725..322ed98e0 100644 --- a/tests/test_roundtrip.rs +++ b/tests/test_roundtrip.rs @@ -137,7 +137,7 @@ fn main() { tree: CompressionTarget::new(Compression::Identity), }; debug!(target: "test_roundtrip", "Encoding."); - let writer = binjs::io::multipart::TreeTokenWriter::new(options.clone()); + let writer = binjs::io::TokenWriterTreeAdapter::new(binjs::io::multipart::TreeTokenWriter::new(options.clone())); let mut serializer = binjs::specialized::es6::io::Serializer::new(writer); serializer.serialize(&ast, &mut IOPath::new()) .expect("Could not encode AST"); @@ -196,7 +196,7 @@ fn main() { // Roundtrip `simple` debug!(target: "test_roundtrip", "Encoding"); - let mut writer = binjs::io::simple::TreeTokenWriter::new(); + let mut writer = binjs::io::TokenWriterTreeAdapter::new(binjs::io::simple::TreeTokenWriter::new()); let mut serializer = binjs::specialized::es6::io::Serializer::new(writer); serializer.serialize(&ast, &mut IOPath::new()) .expect("Could not encode AST"); @@ -230,7 +230,7 @@ fn main() { debug!(target: "test_roundtrip", "Starting multipart round trip for {:?} with options {:?}", entry, options); debug!(target: "test_roundtrip", "Encoding."); options.reset(); - let writer = binjs::io::multipart::TreeTokenWriter::new(options.clone()); + let writer = binjs::io::TokenWriterTreeAdapter::new(binjs::io::multipart::TreeTokenWriter::new(options.clone())); let mut serializer = binjs::specialized::es6::io::Serializer::new(writer); serializer.serialize(&ast, &mut IOPath::new()) .expect("Could not encode AST"); diff --git a/tests/test_simple.rs b/tests/test_simple.rs index 81db88f5d..04b0bb38e 100644 --- a/tests/test_simple.rs +++ b/tests/test_simple.rs @@ -45,7 +45,7 @@ test!(test_simple_tokenization, { visitor.annotate(&mut ast); println!("Encoding sample {}", ast.pretty(2)); - let writer = binjs::io::simple::TreeTokenWriter::new(); + let writer = binjs::io::TokenWriterTreeAdapter::new(binjs::io::simple::TreeTokenWriter::new()); let encoder = binjs::generic::io::encode::Encoder::new(&spec, writer); encoder.encode(&ast)