Skip to content

Commit

Permalink
Test adapters only in terms of Hello World (#5082)
Browse files Browse the repository at this point in the history
Reduces the number of files lying around, and in #5016 it'll be more
difficult to generate hello world blobs with fallback.
  • Loading branch information
robertbastian authored Jun 20, 2024
1 parent b4fe46a commit 0bec39c
Show file tree
Hide file tree
Showing 15 changed files with 95 additions and 65 deletions.
3 changes: 0 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 2 additions & 5 deletions provider/adapters/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ rust-version.workspace = true
version.workspace = true

[dependencies]
icu_locale_core = { workspace = true, features = ["zerovec"] }
icu_locale = { workspace = true }
icu_provider = { workspace = true, features = ["macros"] }
tinystr = { workspace = true, features = ["zerovec"] }
Expand All @@ -28,12 +27,10 @@ serde = { workspace = true, features = ["derive", "alloc"], optional = true }

[dev-dependencies]
icu_provider = { path = "../../provider/core", features = ["macros", "deserialize_json"] }
icu_provider_fs = { path = "../../provider/fs" }
icu_provider_blob = { path = "../../provider/blob" }
icu_locale = { path = "../../components/locale" }
writeable = { path = "../../utils/writeable" }

[features]
std = ["icu_locale_core/std", "icu_provider/std"]
serde = ["dep:serde", "zerovec/serde", "icu_locale_core/serde", "icu_locale/serde", "icu_provider/serde"]
std = ["icu_locale/std", "icu_provider/std"]
serde = ["dep:serde", "zerovec/serde", "icu_locale/serde", "icu_provider/serde"]
datagen = ["std", "serde", "icu_provider/datagen"]
15 changes: 9 additions & 6 deletions provider/adapters/src/fallback/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@ use icu_provider::prelude::*;
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_locale::langid;
/// use icu_provider::prelude::*;
/// use icu_provider::hello_world::*;
/// use icu_provider_adapters::fallback::LocaleFallbackProvider;
///
/// # let provider = icu_provider_blob::BlobDataProvider::try_new_from_static_blob(include_bytes!("../../tests/data/blob.postcard")).unwrap();
/// # let provider = provider.as_deserializing();
/// # let provider = HelloWorldProvider;
/// # struct LocaleFallbackProvider;
/// # impl LocaleFallbackProvider {
/// # fn try_new_unstable(provider: HelloWorldProvider) -> Result<icu_provider_adapters::fallback::LocaleFallbackProvider<HelloWorldProvider>, ()> {
/// # Ok(icu_provider_adapters::fallback::LocaleFallbackProvider::new_with_fallbacker(provider, icu_locale::LocaleFallbacker::new().static_to_owned()))
/// # }
/// # }
///
/// let req = DataRequest {
/// locale: &langid!("ja-JP").into(),
Expand Down Expand Up @@ -119,7 +122,7 @@ impl<P> LocaleFallbackProvider<P> {
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_locale::langid;
/// use icu_locale::LocaleFallbacker;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
Expand Down
10 changes: 5 additions & 5 deletions provider/adapters/src/filter/impls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use super::*;
use alloc::boxed::Box;
use icu_provider::prelude::*;

use icu_locale_core::LanguageIdentifier;
use icu_locale::LanguageIdentifier;

type RequestFilterDataProviderOutput<'a, D> =
RequestFilterDataProvider<D, Box<dyn Fn(DataRequest) -> bool + Sync + 'a>>;
Expand All @@ -24,8 +24,8 @@ where
/// # Examples
///
/// ```
/// use icu_locale_core::LanguageIdentifier;
/// use icu_locale_core::{langid, subtags::language};
/// use icu_locale::LanguageIdentifier;
/// use icu_locale::{langid, subtags::language};
/// use icu_provider::datagen::*;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
Expand Down Expand Up @@ -101,7 +101,7 @@ where
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_locale::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use icu_provider_adapters::filter::Filterable;
Expand Down Expand Up @@ -164,7 +164,7 @@ where
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_locale::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use icu_provider_adapters::filter::Filterable;
Expand Down
2 changes: 1 addition & 1 deletion provider/adapters/src/filter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
//! # Examples
//!
//! ```
//! use icu_locale_core::subtags::language;
//! use icu_locale::subtags::language;
//! use icu_provider::hello_world::*;
//! use icu_provider::prelude::*;
//! use icu_provider_adapters::filter::Filterable;
Expand Down
38 changes: 38 additions & 0 deletions provider/adapters/src/fork/by_error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,24 @@ impl<P0, P1, F> ForkByErrorProvider<P0, P1, F> {
}
}

impl<M, P0, P1, F> DataProvider<M> for ForkByErrorProvider<P0, P1, F>
where
M: DataMarker,
P0: DataProvider<M>,
P1: DataProvider<M>,
F: ForkByErrorPredicate,
{
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
let result = self.0.load(req);
match result {
Ok(ok) => return Ok(ok),
Err(err) if !self.2.test(M::INFO, Some(req), err) => return Err(err),
_ => (),
};
self.1.load(req)
}
}

impl<P0, P1, F> AnyProvider for ForkByErrorProvider<P0, P1, F>
where
P0: AnyProvider,
Expand Down Expand Up @@ -152,6 +170,26 @@ impl<P, F> MultiForkByErrorProvider<P, F> {
}
}

impl<M, P, F> DataProvider<M> for MultiForkByErrorProvider<P, F>
where
M: DataMarker,
P: DataProvider<M>,
F: ForkByErrorPredicate,
{
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
let mut last_error = F::UNIT_ERROR.with_marker(M::INFO);
for provider in self.providers.iter() {
let result = provider.load(req);
match result {
Ok(ok) => return Ok(ok),
Err(err) if !self.predicate.test(M::INFO, Some(req), err) => return Err(err),
Err(err) => last_error = err,
};
}
Err(last_error)
}
}

impl<P, F> AnyProvider for MultiForkByErrorProvider<P, F>
where
P: AnyProvider,
Expand Down
6 changes: 3 additions & 3 deletions provider/adapters/src/fork/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ use predicates::MissingDataMarkerPredicate;
/// Normal usage:
///
/// ```
/// use icu_locale_core::langid;
/// use icu_locale::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use icu_provider_adapters::fork::ForkByMarkerProvider;
Expand Down Expand Up @@ -95,7 +95,7 @@ use predicates::MissingDataMarkerPredicate;
/// Stops at the first provider supporting a marker, even if the locale is not supported:
///
/// ```
/// use icu_locale_core::{subtags::language, langid};
/// use icu_locale::{subtags::language, langid};
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use icu_provider_adapters::filter::Filterable;
Expand Down Expand Up @@ -162,7 +162,7 @@ impl<P0, P1> ForkByMarkerProvider<P0, P1> {
/// # Examples
///
/// ```
/// use icu_locale_core::{subtags::language, langid};
/// use icu_locale::{subtags::language, langid};
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use icu_provider_adapters::filter::Filterable;
Expand Down
28 changes: 16 additions & 12 deletions provider/adapters/src/fork/predicates.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,25 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate {
///
/// # Examples
///
/// Configure a multi-language data provider pointing at two language packs:
///
/// ```
/// use icu_provider_adapters::fork::ForkByErrorProvider;
/// use icu_provider_adapters::fork::predicates::MissingLocalePredicate;
/// use icu_provider_fs::FsDataProvider;
/// use icu_provider::prelude::*;
/// use icu_provider::hello_world::HelloWorldV1Marker;
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_locale::langid;
///
/// struct SingleLocaleProvider(icu_locale::LanguageIdentifier);
/// impl DataProvider<HelloWorldV1Marker> for SingleLocaleProvider {
/// fn load(&self, req: DataRequest) -> Result<DataResponse<HelloWorldV1Marker>, DataError> {
/// if req.locale.get_langid() != self.0 {
/// return Err(DataErrorKind::MissingLocale.with_req(HelloWorldV1Marker::INFO, req));
/// }
/// HelloWorldProvider.load(req)
/// }
/// }
///
/// // The `tests` directory contains two separate "language packs" for Hello World data.
/// let provider_de = FsDataProvider::try_new("tests/data/langtest/de").unwrap();
/// let provider_ro = FsDataProvider::try_new("tests/data/langtest/ro").unwrap();
/// let provider_de = SingleLocaleProvider(langid!("de"));
/// let provider_ro = SingleLocaleProvider(langid!("ro"));
///
/// // Create the forking provider:
/// let provider = ForkByErrorProvider::new_with_predicate(
Expand All @@ -89,7 +95,6 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate {
/// // Test that we can load both "de" and "ro" data:
///
/// let german_hello_world: DataResponse<HelloWorldV1Marker> = provider
/// .as_deserializing()
/// .load(DataRequest {
/// locale: &langid!("de").into(),
/// ..Default::default()
Expand All @@ -99,7 +104,6 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate {
/// assert_eq!("Hallo Welt", german_hello_world.payload.get().message);
///
/// let romanian_hello_world: DataResponse<HelloWorldV1Marker> = provider
/// .as_deserializing()
/// .load(DataRequest {
/// locale: &langid!("ro").into(),
/// ..Default::default()
Expand All @@ -108,10 +112,10 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate {
///
/// assert_eq!("Salut, lume", romanian_hello_world.payload.get().message);
///
/// // We should not be able to load "en" data because it is not in the provider:
/// // We should not be able to load "en" data because it is not in either provider:
///
/// DataProvider::<HelloWorldV1Marker>::load(
/// &provider.as_deserializing(),
/// &provider,
/// DataRequest {
/// locale: &langid!("en").into(),
/// ..Default::default()
Expand Down
Binary file removed provider/adapters/tests/data/blob.postcard
Binary file not shown.

This file was deleted.

3 changes: 0 additions & 3 deletions provider/adapters/tests/data/langtest/de/manifest.json

This file was deleted.

This file was deleted.

3 changes: 0 additions & 3 deletions provider/adapters/tests/data/langtest/ro/manifest.json

This file was deleted.

7 changes: 3 additions & 4 deletions tools/make/data.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,17 @@ exit_on_error true
exec --fail-on-error cargo build -p icu4x-datagen --no-default-features --features fs_exporter,blob_exporter,baked_exporter,networking,provider
# Filesystem
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format dir --syntax json --out provider/fs/tests/data/json --overwrite
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format dir --syntax bincode --out provider/fs/tests/data/bincode --overwrite
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format dir --syntax postcard --out provider/fs/tests/data/postcard --overwrite
# Blob
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format blob --overwrite --out provider/blob/tests/data/v1.postcard
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format blob2 --overwrite --out provider/blob/tests/data/v2.postcard
# Baked
exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --locales full --format mod --pretty --overwrite --no-internal-fallback --deduplication none --out provider/baked/tests/data
# exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker "fallback/likelysubtags@1" "fallback/parents@1" "fallback/supplement/co@1" --locales full --format blob --out provider/adapters/tests/data/blob.postcard --overwrite
# exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --without-fallback --locales de --format dir --syntax json --out provider/adapters/tests/data/langtest/de --overwrite
# exec --fail-on-error target/debug/icu4x-datagen --markers HelloWorldV1Marker --without-fallback --locales ro --format dir --syntax json --out provider/adapters/tests/data/langtest/ro --overwrite
'''

[tasks.testdata-check]
Expand Down
36 changes: 18 additions & 18 deletions tutorials/data_provider.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use icu_provider_adapters::fork::MultiForkByErrorProvider;
use icu_provider_adapters::fork::predicates::MissingLocalePredicate;
use icu_provider_blob::BlobDataProvider;
use icu_provider_fs::FsDataProvider;
use icu_provider::DataLocale;
use icu_provider::hello_world::HelloWorldFormatter;
use icu_provider::prelude::*;
use icu_provider::hello_world::*;
use icu::locale::locale;
use icu::locale::subtags::Language;
use std::path::Path;
Expand All @@ -27,28 +27,28 @@ let mut provider = MultiForkByErrorProvider::new_with_predicate(
MissingLocalePredicate
);

// Pretend we're loading these from the network or somewhere.
struct SingleLocaleProvider(DataLocale);

impl DataProvider<HelloWorldV1Marker> for SingleLocaleProvider {
fn load(&self, req: DataRequest) -> Result<DataResponse<HelloWorldV1Marker>, DataError> {
if *req.locale != self.0 {
return Err(DataErrorKind::MissingLocale.with_req(HelloWorldV1Marker::INFO, req));
}
HelloWorldProvider.load(req)
}
}

// Helper function to add data into the growable provider on demand:
let mut get_hello_world_formatter = |loc: &DataLocale| {
// Try to create the formatter a first time with data that has already been loaded.
if let Ok(formatter) = HelloWorldFormatter::try_new_with_buffer_provider(&provider, loc) {
if let Ok(formatter) = HelloWorldFormatter::try_new_unstable(&provider, loc) {
return formatter;
}

// We failed to create the formatter. Load more data for the language.
// Note: This assumes data is split by language subtag, which may or may not be the best
// strategy for all use cases.
let path_buf =
Path::new("../../provider/adapters/tests/data/langtest")
.join(loc.language().as_str());
let lang_provider = match FsDataProvider::try_new(&path_buf) {
Ok(p) => p,
Err(e) => panic!("Language not available? {:?}", e)
};
println!("Successfully loaded: {:?}", loc);

// Add the data to the growable provider and try creating the formatter a second time.
provider.push(lang_provider);
HelloWorldFormatter::try_new_with_buffer_provider(&provider, loc)
// We failed to create the formatter. Load more data for the language and try creating the formatter a second time.
provider.push(SingleLocaleProvider(loc.clone()));
HelloWorldFormatter::try_new_unstable(&provider, loc)
.expect("Language data should now be available")
};

Expand Down

0 comments on commit 0bec39c

Please sign in to comment.