h

provider/baked/src/export.rs

@@ -154,9 +154,12 @@ pub struct BakedExporter {
     pretty: bool,
     use_separate_crates: bool,
     use_internal_fallback: bool,
-    // Temporary storage for put_payload: marker -> (bake -> {data id})
+    // Temporary storage for put_payload: marker -> (payload -> {data id})
     data: Mutex<
-        HashMap<DataMarkerInfo, BTreeMap<SyncTokenStream, HashSet<DataIdentifierCow<'static>>>>,
+        HashMap<
+            DataMarkerInfo,
+            HashMap<DataPayload<ExportMarker>, HashSet<DataIdentifierCow<'static>>>,
+        >,
     >,
     /// (marker, file name) pairs to wire up in mod.rs. This is populated by `flush` and consumed by `close`.
     impl_data: Mutex<BTreeMap<DataMarkerInfo, SyncTokenStream>>,
@@ -355,14 +358,12 @@ impl DataExporter for BakedExporter {
         id: DataIdentifierBorrowed,
         payload: &DataPayload<ExportMarker>,
     ) -> Result<(), DataError> {
-        let payload = payload.tokenize(&self.dependencies);
-        let payload = payload.to_string();
         self.data
             .lock()
             .expect("poison")
             .entry(marker)
             .or_default()
-            .entry(payload)
+            .entry(payload.clone())
             .or_default()
             .insert(id.into_owned());
         Ok(())
@@ -466,9 +467,7 @@ impl DataExporter for BakedExporter {
 
             let ids_to_idents = deduplicated_values
                 .iter()
-                .flat_map(|(bake, ids)| {
-                    let bake = bake.parse::<TokenStream>().unwrap();
-
+                .flat_map(|(payload, ids)| {
                     let mut idents = ids
                         .iter()
                         .map(|id| {
@@ -484,10 +483,12 @@ impl DataExporter for BakedExporter {
                                 .collect::<String>()
                         })
                         .collect::<Vec<_>>();
-                    idents.sort();
-                    let ident = proc_macro2::Ident::new(&idents[0], proc_macro2::Span::call_site());
+                    let ident = proc_macro2::Ident::new(
+                        &idents.select_nth_unstable(0).1,
+                        proc_macro2::Span::call_site(),
+                    );
 
-                    idents_to_bakes.push((ident.clone(), bake));
+                    idents_to_bakes.push((ident.clone(), payload.tokenize(&self.dependencies)));
                     ids.iter().map(move |id| (id.clone(), ident.clone()))
                 })
                 .collect();

provider/core/Cargo.toml

@@ -70,7 +70,7 @@ deserialize_bincode_1 = ["serde", "dep:bincode", "std"]
 deserialize_postcard_1 = ["serde", "dep:postcard"]
 
 # Dependencies for running data generation
-export = ["serde", "dep:erased-serde", "dep:databake", "std", "sync"]
+export = ["serde", "dep:erased-serde", "dep:databake", "std", "sync", "dep:postcard"]
 
 [package.metadata.cargo-all-features]
 denylist = ["macros"]

provider/core/src/export/payload.rs

@@ -6,7 +6,7 @@ use core::any::Any;
 
 use crate::dynutil::UpcastDataPayload;
 use crate::prelude::*;
-use alloc::boxed::Box;
+use alloc::sync::Arc;
 use databake::{Bake, CrateEnv, TokenStream};
 use yoke::trait_hack::YokeTraitHack;
 use yoke::*;
@@ -62,9 +62,9 @@ where
 }
 
 #[doc(hidden)] // macro
-#[derive(yoke::Yokeable)]
+#[derive(yoke::Yokeable, Clone)]
 pub struct ExportBox {
-    payload: Box<dyn ExportableDataPayload + Sync + Send>,
+    payload: Arc<dyn ExportableDataPayload + Sync + Send>,
 }
 
 impl PartialEq for ExportBox {
@@ -73,6 +73,8 @@ impl PartialEq for ExportBox {
     }
 }
 
+impl Eq for ExportBox {}
+
 impl core::fmt::Debug for ExportBox {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         f.debug_struct("ExportBox")
@@ -90,7 +92,7 @@ where
 {
     fn upcast(other: DataPayload<M>) -> DataPayload<ExportMarker> {
         DataPayload::from_owned(ExportBox {
-            payload: Box::new(other),
+            payload: Arc::new(other),
         })
     }
 }
@@ -165,6 +167,49 @@ impl DataPayload<ExportMarker> {
     pub fn tokenize(&self, env: &CrateEnv) -> TokenStream {
         self.get().payload.bake_yoke(env)
     }
+
+    /// Returns the data size using postcard encoding
+    pub fn postcard_size(&self) -> usize {
+        use postcard::ser_flavors::{Flavor, Size};
+        let mut serializer = postcard::Serializer {
+            output: Size::default(),
+        };
+        self.get()
+            .payload
+            .serialize_yoke(&mut <dyn erased_serde::Serializer>::erase(&mut serializer))
+            .unwrap();
+
+        serializer.output.finalize().unwrap()
+    }
+}
+
+impl core::hash::Hash for DataPayload<ExportMarker> {
+    fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
+        use postcard::ser_flavors::Flavor;
+
+        struct HashFlavor<'a, H>(&'a mut H);
+        impl<'a, H: core::hash::Hasher> Flavor for HashFlavor<'a, H> {
+            type Output = ();
+
+            fn try_push(&mut self, data: u8) -> postcard::Result<()> {
+                self.0.write_u8(data);
+                Ok(())
+            }
+
+            fn finalize(self) -> postcard::Result<Self::Output> {
+                Ok(())
+            }
+        }
+
+        let _infallible =
+            self.get()
+                .payload
+                .serialize_yoke(&mut <dyn erased_serde::Serializer>::erase(
+                    &mut postcard::Serializer {
+                        output: HashFlavor(state),
+                    },
+                ));
+    }
 }
 
 /// Marker type for [`ExportBox`].

tools/bakeddata-scripts/Cargo.toml

@@ -11,7 +11,7 @@ edition = "2021"
 [dependencies]
 icu = { workspace = true, features = ["experimental"] }
 icu_provider = { workspace = true }
-icu_datagen = { workspace = true, features = ["baked_exporter", "fs_exporter", "rayon"] }
+icu_datagen = { workspace = true, features = ["baked_exporter", "rayon"] }
 icu_datagen_bikeshed = { workspace = true, features = ["networking", "experimental", "use_wasm"] }
 
 log = { workspace = true }

tools/bakeddata-scripts/src/main.rs

@@ -5,8 +5,6 @@
 extern crate icu_datagen;
 
 use icu_datagen::baked_exporter;
-use icu_datagen::fs_exporter;
-use icu_datagen::fs_exporter::serializers::AbstractSerializer;
 use icu_datagen::prelude::*;
 use icu_datagen_bikeshed::{CoverageLevel, DatagenProvider};
 use icu_provider::export::*;
@@ -232,22 +230,17 @@ impl<F: Write + Send + Sync> DataExporter for StatisticsExporter<F> {
         &self,
         marker: DataMarkerInfo,
         id: DataIdentifierBorrowed,
-        payload_before: &DataPayload<ExportMarker>,
+        payload: &DataPayload<ExportMarker>,
     ) -> Result<(), DataError> {
-        let mut serialized = vec![];
-
-        fs_exporter::serializers::Postcard::new(Default::default())
-            .serialize(payload_before, &mut serialized)?;
-
-        let size = serialized.len();
+        let size = payload.postcard_size();
 
         // We're using SipHash, which is deprecated, but we want a stable hasher
         // (we're fine with it not being cryptographically secure since we're just using it to track diffs)
         #[allow(deprecated)]
         use std::hash::{Hash, Hasher, SipHasher};
         #[allow(deprecated)]
         let mut hasher = SipHasher::new();
-        serialized.iter().for_each(|b| b.hash(&mut hasher));
+        payload.hash(&mut hasher);
         let hash = hasher.finish();
 
         let mut data = self.data.lock().expect("poison");