Skip to content

Commit

Permalink
Remove duplicate S3 test helper functions (awslabs#664)
Browse files Browse the repository at this point in the history
Signed-off-by: Alessandro Passaro <[email protected]>
  • Loading branch information
passaro authored Dec 6, 2023
1 parent 6fc30b9 commit e9d7043
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 162 deletions.
87 changes: 3 additions & 84 deletions mountpoint-s3/tests/common/fuse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,14 @@ use std::fs::ReadDir;
use std::path::Path;
use std::sync::Arc;

use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_sts::config::Region;
use fuser::{BackgroundSession, MountOption, Session};
use futures::Future;
use mountpoint_s3::data_cache::DataCache;
use mountpoint_s3::fuse::S3FuseFilesystem;
use mountpoint_s3::prefetch::{Prefetch, PrefetcherConfig};
use mountpoint_s3::prefix::Prefix;
use mountpoint_s3::S3FilesystemConfig;
use mountpoint_s3_client::types::PutObjectParams;
use mountpoint_s3_client::ObjectClient;
use rand::RngCore;
use rand_chacha::rand_core::OsRng;
use tempfile::TempDir;

pub trait TestClient: Send {
Expand Down Expand Up @@ -248,8 +243,6 @@ pub mod mock_session {
pub mod s3_session {
use super::*;

use std::future::Future;

use aws_sdk_s3::config::Region;
use aws_sdk_s3::operation::head_object::HeadObjectError;
use aws_sdk_s3::primitives::ByteStream;
Expand All @@ -259,6 +252,8 @@ pub mod s3_session {
use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig};
use mountpoint_s3_client::S3CrtClient;

use crate::common::s3::{get_s3express_endpoint, get_test_bucket_and_prefix, get_test_region, tokio_block_on};

/// Create a FUSE mount backed by a real S3 client
pub fn new(test_name: &str, test_config: TestSessionConfig) -> (TempDir, BackgroundSession, TestClientBox) {
let mount_dir = tempfile::tempdir().unwrap();
Expand Down Expand Up @@ -330,23 +325,14 @@ pub mod s3_session {
}

async fn get_test_sdk_client(region: &str) -> aws_sdk_s3::Client {
let mut config = aws_config::from_env().region(Region::new(get_test_region()));
let mut config = aws_config::from_env().region(Region::new(region.to_owned()));
if cfg!(feature = "s3express_tests") {
config = config.endpoint_url(get_s3express_endpoint());
}
let config = config.load().await;
aws_sdk_s3::Client::new(&config)
}

fn tokio_block_on<F: Future>(future: F) -> F::Output {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap();
runtime.block_on(future)
}

struct SDKTestClient {
prefix: String,
bucket: String,
Expand Down Expand Up @@ -513,70 +499,3 @@ pub fn read_dir_to_entry_names(read_dir_iter: ReadDir) -> Vec<String> {
})
.collect::<Vec<_>>()
}

pub fn get_test_bucket_and_prefix(test_name: &str) -> (String, String) {
let bucket = if cfg!(feature = "s3express_tests") {
std::env::var("S3_EXPRESS_ONE_ZONE_BUCKET_NAME")
.expect("Set S3_EXPRESS_ONE_ZONE_BUCKET_NAME to run integration tests")
} else {
std::env::var("S3_BUCKET_NAME").expect("Set S3_BUCKET_NAME to run integration tests")
};

// Generate a random nonce to make sure this prefix is truly unique
let nonce = OsRng.next_u64();

// Prefix always has a trailing "/" to keep meaning in sync with the S3 API.
let prefix = std::env::var("S3_BUCKET_TEST_PREFIX").unwrap_or(String::from("mountpoint-test/"));
assert!(prefix.ends_with('/'), "S3_BUCKET_TEST_PREFIX should end in '/'");

let prefix = format!("{prefix}{test_name}/{nonce}/");

(bucket, prefix)
}

pub fn get_test_bucket_forbidden() -> String {
std::env::var("S3_FORBIDDEN_BUCKET_NAME").expect("Set S3_FORBIDDEN_BUCKET_NAME to run integration tests")
}

pub fn get_test_region() -> String {
std::env::var("S3_REGION").expect("Set S3_REGION to run integration tests")
}

pub fn get_subsession_iam_role() -> String {
std::env::var("S3_SUBSESSION_IAM_ROLE").expect("Set S3_SUBSESSION_IAM_ROLE to run integration tests")
}

pub fn get_s3express_endpoint() -> String {
std::env::var("S3_EXPRESS_ONE_ZONE_ENDPOINT").expect("Set S3_EXPRESS_ONE_ZONE_ENDPOINT to run integration tests")
}

pub fn create_objects(bucket: &str, prefix: &str, region: &str, key: &str, value: &[u8]) {
let mut config = aws_config::from_env().region(Region::new(get_test_region()));
if cfg!(feature = "s3express_tests") {
config = config.endpoint_url(get_s3express_endpoint());
}
let config = tokio_block_on(config.load());
let sdk_client = aws_sdk_s3::Client::new(&config);
let full_key = format!("{prefix}{key}");
let mut request = sdk_client.put_object();
if cfg!(not(feature = "s3express_tests")) {
request = request.bucket(bucket);
}
tokio_block_on(async move {
request
.key(full_key)
.body(ByteStream::from(value.to_vec()))
.send()
.await
.unwrap()
});
}

pub fn tokio_block_on<F: Future>(future: F) -> F::Output {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap();
runtime.block_on(future)
}
79 changes: 5 additions & 74 deletions mountpoint-s3/tests/common/mod.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
//! Functions and types shared across integration test modules.
//! Allow for unused code since this is included independently in each module.
#![allow(unused)]
//! Allow for unused items since this is included independently in each module.
#![allow(dead_code)]

#[cfg(feature = "fuse_tests")]
pub mod fuse;

use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_sts::config::Region;
#[cfg(feature = "s3_tests")]
pub mod s3;

use fuser::{FileAttr, FileType};
use futures::executor::ThreadPool;
use futures::Future;
use mountpoint_s3::fs::{self, DirectoryEntry, DirectoryReplier, ReadReplier, ToErrno};
use mountpoint_s3::prefetch::{default_prefetch, DefaultPrefetcher};
use mountpoint_s3::prefix::Prefix;
use mountpoint_s3::{S3Filesystem, S3FilesystemConfig};
use mountpoint_s3_client::mock_client::{MockClient, MockClientConfig};
use mountpoint_s3_client::ObjectClient;
use mountpoint_s3_crt::common::rust_log_adapter::RustLogAdapter;
use rand::RngCore;
use rand_chacha::rand_core::OsRng;
use std::collections::VecDeque;
use std::sync::Arc;

Expand Down Expand Up @@ -54,73 +52,6 @@ where
S3Filesystem::new(client, prefetcher, bucket, prefix, config)
}

pub fn get_test_bucket_and_prefix(test_name: &str) -> (String, String) {
let bucket = if cfg!(feature = "s3express_tests") {
std::env::var("S3_EXPRESS_ONE_ZONE_BUCKET_NAME")
.expect("Set S3_EXPRESS_ONE_ZONE_BUCKET_NAME to run integration tests")
} else {
std::env::var("S3_BUCKET_NAME").expect("Set S3_BUCKET_NAME to run integration tests")
};

// Generate a random nonce to make sure this prefix is truly unique
let nonce = OsRng.next_u64();

// Prefix always has a trailing "/" to keep meaning in sync with the S3 API.
let prefix = std::env::var("S3_BUCKET_TEST_PREFIX").unwrap_or(String::from("mountpoint-test/"));
assert!(prefix.ends_with('/'), "S3_BUCKET_TEST_PREFIX should end in '/'");

let prefix = format!("{prefix}{test_name}/{nonce}/");

(bucket, prefix)
}

pub fn get_test_bucket_forbidden() -> String {
std::env::var("S3_FORBIDDEN_BUCKET_NAME").expect("Set S3_FORBIDDEN_BUCKET_NAME to run integration tests")
}

pub fn get_test_region() -> String {
std::env::var("S3_REGION").expect("Set S3_REGION to run integration tests")
}

pub fn get_subsession_iam_role() -> String {
std::env::var("S3_SUBSESSION_IAM_ROLE").expect("Set S3_SUBSESSION_IAM_ROLE to run integration tests")
}

pub fn get_s3express_endpoint() -> String {
std::env::var("S3_EXPRESS_ONE_ZONE_ENDPOINT").expect("Set S3_EXPRESS_ONE_ZONE_ENDPOINT to run integration tests")
}

pub fn create_objects(bucket: &str, prefix: &str, region: &str, key: &str, value: &[u8]) {
let mut config = aws_config::from_env().region(Region::new(region.to_string()));
if cfg!(feature = "s3express_tests") {
config = config.endpoint_url(get_s3express_endpoint());
}
let config = tokio_block_on(config.load());
let sdk_client = aws_sdk_s3::Client::new(&config);
let full_key = format!("{prefix}{key}");
tokio_block_on(async move {
let mut request = sdk_client.put_object();
if cfg!(not(feature = "s3express_tests")) {
request = request.bucket(bucket);
}
request
.key(full_key)
.body(ByteStream::from(value.to_vec()))
.send()
.await
.unwrap()
});
}

pub fn tokio_block_on<F: Future>(future: F) -> F::Output {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap();
runtime.block_on(future)
}

#[track_caller]
pub fn assert_attr(attr: FileAttr, ftype: FileType, size: u64, uid: u32, gid: u32, perm: u16) {
assert_eq!(attr.kind, ftype);
Expand Down
72 changes: 72 additions & 0 deletions mountpoint-s3/tests/common/s3.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_sts::config::Region;
use futures::Future;
use rand::RngCore;
use rand_chacha::rand_core::OsRng;

pub fn get_test_bucket_and_prefix(test_name: &str) -> (String, String) {
let bucket = if cfg!(feature = "s3express_tests") {
std::env::var("S3_EXPRESS_ONE_ZONE_BUCKET_NAME")
.expect("Set S3_EXPRESS_ONE_ZONE_BUCKET_NAME to run integration tests")
} else {
std::env::var("S3_BUCKET_NAME").expect("Set S3_BUCKET_NAME to run integration tests")
};

// Generate a random nonce to make sure this prefix is truly unique
let nonce = OsRng.next_u64();

// Prefix always has a trailing "/" to keep meaning in sync with the S3 API.
let prefix = std::env::var("S3_BUCKET_TEST_PREFIX").unwrap_or(String::from("mountpoint-test/"));
assert!(prefix.ends_with('/'), "S3_BUCKET_TEST_PREFIX should end in '/'");

let prefix = format!("{prefix}{test_name}/{nonce}/");

(bucket, prefix)
}

pub fn get_test_bucket_forbidden() -> String {
std::env::var("S3_FORBIDDEN_BUCKET_NAME").expect("Set S3_FORBIDDEN_BUCKET_NAME to run integration tests")
}

pub fn get_test_region() -> String {
std::env::var("S3_REGION").expect("Set S3_REGION to run integration tests")
}

pub fn get_subsession_iam_role() -> String {
std::env::var("S3_SUBSESSION_IAM_ROLE").expect("Set S3_SUBSESSION_IAM_ROLE to run integration tests")
}

pub fn get_s3express_endpoint() -> String {
std::env::var("S3_EXPRESS_ONE_ZONE_ENDPOINT").expect("Set S3_EXPRESS_ONE_ZONE_ENDPOINT to run integration tests")
}

pub fn create_objects(bucket: &str, prefix: &str, region: &str, key: &str, value: &[u8]) {
let mut config = aws_config::from_env().region(Region::new(region.to_string()));
if cfg!(feature = "s3express_tests") {
config = config.endpoint_url(get_s3express_endpoint());
}
let config = tokio_block_on(config.load());
let sdk_client = aws_sdk_s3::Client::new(&config);
let full_key = format!("{prefix}{key}");
tokio_block_on(async move {
let mut request = sdk_client.put_object();
if cfg!(not(feature = "s3express_tests")) {
request = request.bucket(bucket);
}
request
.key(full_key)
.body(ByteStream::from(value.to_vec()))
.send()
.await
.unwrap()
});
}

pub fn tokio_block_on<F: Future>(future: F) -> F::Output {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap();
runtime.block_on(future)
}
7 changes: 3 additions & 4 deletions mountpoint-s3/tests/fuse_tests/fork_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,10 @@ use std::process::Stdio;
use std::{path::PathBuf, process::Command};
use test_case::test_case;

use crate::common::fuse::{
create_objects, get_test_bucket_and_prefix, get_test_bucket_forbidden, get_test_region, read_dir_to_entry_names,
};
use crate::common::fuse::read_dir_to_entry_names;
use crate::common::s3::{create_objects, get_test_bucket_and_prefix, get_test_bucket_forbidden, get_test_region};
#[cfg(not(feature = "s3express_tests"))]
use crate::common::fuse::{get_subsession_iam_role, tokio_block_on};
use crate::common::s3::{get_subsession_iam_role, tokio_block_on};

const MAX_WAIT_DURATION: std::time::Duration = std::time::Duration::from_secs(10);

Expand Down

0 comments on commit e9d7043

Please sign in to comment.