Make testdata generation a unit test so it can access cfg(test) code (#…

…4293)

Preparing for #4280

provider/datagen/Cargo.toml

@@ -143,11 +143,6 @@ name = "icu4x-datagen"
 path = "src/bin/datagen/mod.rs"
 required-features = ["bin"]
 
-[[test]]
-name = "make-testdata"
-path = "tests/make-testdata.rs"
-required-features = ["fs_exporter", "use_wasm"]
-
 [package.metadata.cargo-all-features]
 # We don't need working CPT builders for check
 skip_feature_sets = [["use_icu4c"], ["use_wasm"]]

provider/datagen/src/lib.rs

@@ -141,6 +141,9 @@ mod registry;
 mod source;
 mod transform;
 
+#[cfg(test)]
+mod tests;
+
 pub use driver::DatagenDriver;
 pub use provider::DatagenProvider;
 #[doc(hidden)] // for CLI serde

provider/datagen/src/registry.rs

@@ -120,7 +120,7 @@ macro_rules! registry {
             unreachable!("unregistered key {key:?}")
         }
 
-        #[doc(hidden)]
+        #[cfg(test)]
         pub fn deserialize_and_measure<Measurement>(key: DataKey, buf: DataPayload<BufferMarker>, measure: impl Fn() -> Measurement) -> Result<(Measurement, DataPayload<icu_provider::datagen::ExportMarker>), DataError> {
             if key.path() == icu_provider::hello_world::HelloWorldV1Marker::KEY.path() {
                 let deserialized: DataPayload<icu_provider::hello_world::HelloWorldV1Marker> = buf.into_deserialized(icu_provider::buf::BufferFormat::Postcard1)?;

provider/datagen/tests/make-testdata.rs → provider/datagen/src/tests/make_testdata.rs

@@ -2,12 +2,12 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+use crate::prelude::*;
 use crlify::BufWriterWithLineEndingFix;
-use icu_datagen::fs_exporter::serializers::Json;
-use icu_datagen::fs_exporter::*;
-use icu_datagen::prelude::*;
 use icu_provider::datagen::*;
 use icu_provider::prelude::*;
+use icu_provider_fs::export::serializers::Json;
+use icu_provider_fs::export::*;
 use std::alloc::{GlobalAlloc, Layout, System};
 use std::cell::Cell;
 use std::collections::{BTreeMap, BTreeSet};
@@ -16,7 +16,7 @@ use std::io::Write;
 use std::path::Path;
 use std::sync::Mutex;
 
-include!("locales.rs.data");
+include!("../../tests/locales.rs.data");
 
 #[test]
 #[ignore] // has side effects, run manually
@@ -29,14 +29,6 @@ fn generate_json_and_verify_postcard() {
 
     let data_root = Path::new(concat!(core::env!("CARGO_MANIFEST_DIR"), "/tests/data/"));
 
-    let source = DatagenProvider::new_custom()
-        .with_cldr(data_root.join("cldr"))
-        .unwrap()
-        .with_icuexport(data_root.join("icuexport"))
-        .unwrap()
-        .with_segmenter_lstm(data_root.join("lstm"))
-        .unwrap();
-
     let json_out = Box::new(
         FilesystemExporter::try_new(Box::new(Json::pretty()), {
             let mut options = ExporterOptions::default();
@@ -58,13 +50,16 @@ fn generate_json_and_verify_postcard() {
     });
 
     DatagenDriver::new()
-        .with_keys(icu_datagen::all_keys())
+        .with_keys(crate::all_keys())
         .with_locales(LOCALES.iter().cloned())
         .with_segmenter_models([
             "thaidict".into(),
             "Thai_codepoints_exclusive_model4_heavy".into(),
         ])
-        .export(&source, MultiExporter::new(vec![json_out, postcard_out]))
+        .export(
+            &DatagenProvider::new_testing(),
+            MultiExporter::new(vec![json_out, postcard_out]),
+        )
         .unwrap();
 }
 
@@ -132,12 +127,9 @@ impl DataExporter for PostcardTestingExporter {
 
         MeasuringAllocator::start_measure();
 
-        let ((allocated, deallocated), payload_after) = icu_datagen::deserialize_and_measure(
-            key,
-            buffer_payload,
-            MeasuringAllocator::end_measure,
-        )
-        .unwrap();
+        let ((allocated, deallocated), payload_after) =
+            crate::deserialize_and_measure(key, buffer_payload, MeasuringAllocator::end_measure)
+                .unwrap();
 
         if payload_before != &payload_after {
             self.rountrip_errors

provider/datagen/src/tests/mod.rs

@@ -0,0 +1,10 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+//! Module for tests that need to access `#[cfg(test)]` code and dependencies.
+//!
+//! Most tests should either be in-module unit tests or integration tests.
+
+#[cfg(all(feature = "fs_exporter", feature = "use_wasm"))]
+mod make_testdata;

tools/make/data.toml

@@ -22,12 +22,12 @@ command = "cargo"
 args = [
     "test",
     "-p=icu_datagen",
-    "--test=make-testdata",
     "--no-default-features",
     "--features=fs_exporter,use_wasm,rayon,experimental_components",
     "--",
     "--ignored",
     "--nocapture",
+    "generate_json_and_verify_postcard",
 ]
 
 [tasks.testdata-hello-world]