diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 0c264e1a8..3c4be164f 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -438,6 +438,7 @@ dependencies = [ "rand", "serde", "serde_json", + "serde_plain", "simple-settings-plugin", "simplelog", "snafu", diff --git a/sources/api/apiserver/Cargo.toml b/sources/api/apiserver/Cargo.toml index a6c87e287..1ba38da35 100644 --- a/sources/api/apiserver/Cargo.toml +++ b/sources/api/apiserver/Cargo.toml @@ -27,6 +27,7 @@ num.workspace = true rand = { workspace = true, features = ["default"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +serde_plain.workspace = true simplelog.workspace = true snafu.workspace = true thar-be-updates.workspace = true diff --git a/sources/api/apiserver/src/server/controller.rs b/sources/api/apiserver/src/server/controller.rs index e48b7cfa7..41d798d66 100644 --- a/sources/api/apiserver/src/server/controller.rs +++ b/sources/api/apiserver/src/server/controller.rs @@ -3,6 +3,7 @@ use bottlerocket_release::BottlerocketRelease; use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt}; use std::collections::{HashMap, HashSet}; use std::io::Write; @@ -10,10 +11,13 @@ use std::process::{Command, Stdio}; use crate::server::error::{self, Result}; use actix_web::HttpResponse; +use datastore::constraints_check::{ApprovedWrite, ConstraintCheckResult}; use datastore::deserialization::{from_map, from_map_with_prefix}; use datastore::serialization::to_pairs_with_prefix; -use datastore::{deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value}; -use model::{ConfigurationFiles, Services, Settings}; +use datastore::{ + deserialize_scalar, serialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value, +}; +use model::{ConfigurationFiles, Services, Settings, Strength}; use num::FromPrimitive; use std::os::unix::process::ExitStatusExt; use thar_be_updates::error::TbuErrorStatus; @@ -44,6 +48,51 @@ where .map(|maybe_settings| maybe_settings.unwrap_or_default()) } +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct SettingsMetadata { + pub(crate) inner: HashMap>, +} + +impl From>> for SettingsMetadata { + fn from(transaction_metadata: HashMap>) -> Self { + let mut metadata = HashMap::new(); + for (key, value) in transaction_metadata { + let mut inner_map = HashMap::new(); + for (inner_key, inner_value) in value { + inner_map.insert(inner_key.name().clone(), inner_value); + } + metadata.insert(key.name().clone(), inner_map); + } + + SettingsMetadata { inner: metadata } + } +} + +/// Gets the metadata for metadata_key_name in the given transaction +/// Returns all metadata if metadata_key_name is None +pub(crate) fn get_transaction_metadata( + datastore: &D, + transaction: S, + metadata_key_name: Option, +) -> Result +where + D: DataStore, + S: Into, +{ + let pending = Committed::Pending { + tx: transaction.into(), + }; + + let metadata = datastore + .get_metadata_prefix("settings.", &pending, &metadata_key_name) + .with_context(|_| error::DataStoreSnafu { + op: format!("get_metadata_prefix '{}' for {:?}", "settings.", pending), + })?; + + Ok(SettingsMetadata::from(metadata)) +} + /// Deletes the transaction from the data store, removing any uncommitted settings under that /// transaction name. pub(crate) fn delete_transaction( @@ -364,6 +413,7 @@ pub(crate) fn set_settings( datastore: &mut D, settings: &Settings, transaction: &str, + strength: Strength, ) -> Result<()> { trace!("Serializing Settings to write to data store"); let settings_json = serde_json::to_value(settings).context(error::SettingsToJsonSnafu)?; @@ -372,6 +422,96 @@ pub(crate) fn set_settings( let pending = Committed::Pending { tx: transaction.into(), }; + + info!("Writing Metadata to data store"); + match strength { + Strength::Strong => { + // Get keys in the request + let keys: HashSet<&str> = pairs.iter().map(|pair| pair.0.name().as_str()).collect(); + // Get strength metadata for the keys from live + let committed_strength_live = get_metadata_for_data_keys(datastore, "strength", &keys)?; + + // Change the weak strength to strong if the committed strength is weak and requested strength is strong + for (key, value) in committed_strength_live { + // if the strength is weak then we need to change it to strong + if value == Strength::Weak.to_string() { + let data_key = + Key::new(KeyType::Data, key.clone()).context(error::NewKeySnafu { + key_type: "data", + name: key.clone(), + })?; + + let metadata_key_strength = + Key::new(KeyType::Meta, "strength").context(error::NewKeySnafu { + key_type: "meta", + name: "strength", + })?; // change this to name as strength and value as weak or strong + + let metadata_value = datastore::serialize_scalar::<_, ScalarError>( + &Strength::Strong.to_string(), + ) + .with_context(|_| error::SerializeSnafu {})?; + + datastore + .set_metadata(&metadata_key_strength, &data_key, metadata_value, &pending) + .context(error::DataStoreSnafu { + op: "Change strength metadata key to strong", + })?; + } + } + } + Strength::Weak => { + for key in pairs.keys() { + // The get key funtion returns Ok(None) in case if the path does not exist + // and error if some path exist and some error occurred in fetching + // Hence we we will return error in case of error + // from get key function and continue to add/change to weak key + // if the value is None. + let value = datastore + .get_key(key, &Committed::Live) + .context(error::DataStoreSnafu { op: "get_key" })?; + + // Get metadata value for the key + // If strength does not exist this hashmap will be empty + // and if strength exist this hashmap will return HashMap + let mut keys_to_get_metadata: HashSet<&str> = HashSet::new(); + keys_to_get_metadata.insert(key.name().as_str()); + let strength_pair = + get_metadata_for_data_keys(datastore, "strength", &keys_to_get_metadata)?; + + let is_setting_strong = strength_pair.is_empty() + || strength_pair.get(key.name().as_str()) + == Some(&serde_json::Value::String(Strength::Strong.to_string())); + + // We need to log that we are not changing the strength from strong to weak + // and continue for other settings. + if value.is_some() && is_setting_strong { + warn!("Trying to change the strength from strong to weak for key: {}, Operation ignored", key.name()); + continue; + } + + // If the strength and setting both does not exist and requested strength is weak + // Set strength metadata. + let metadata_key = + Key::new(KeyType::Meta, "strength").context(error::NewKeySnafu { + key_type: "meta", + name: "strength", + })?; + + let metadata_value = + datastore::serialize_scalar::<_, ScalarError>(&Strength::Weak.to_string()) + .with_context(|_| error::SerializeSnafu {})?; + + datastore + .set_metadata(&metadata_key, key, metadata_value, &pending) + .context(error::DataStoreSnafu { + op: "create strength metadata key as weak", + })?; + } + } + }; + + info!("Writing Settings to data store: {:?}", pairs); datastore .set_keys(&pairs, &pending) .context(error::DataStoreSnafu { op: "set_keys" }) @@ -398,7 +538,7 @@ pub(crate) fn get_metadata_for_data_keys>( key_type: "data", name: *data_key_str, })?; - let value_str = match datastore.get_metadata(&md_key, &data_key) { + let value_str = match datastore.get_metadata(&md_key, &data_key, &Committed::Live) { Ok(Some(v)) => v, // TODO: confirm we want to skip requested keys if not populated, or error Ok(None) => continue, @@ -428,7 +568,7 @@ pub(crate) fn get_metadata_for_all_data_keys>( ) -> Result> { trace!("Getting metadata '{}'", md_key_str.as_ref()); let meta_map = datastore - .get_metadata_prefix("", &Some(md_key_str)) + .get_metadata_prefix("", &Committed::Live, &Some(md_key_str)) .context(error::DataStoreSnafu { op: "get_metadata_prefix", })?; @@ -449,13 +589,135 @@ pub(crate) fn get_metadata_for_all_data_keys>( Ok(result) } +// Parses and validates the settings and metadata in pending transaction and +// returns the constraint check result containing approved settings and metadata to +// commit to live transaction. +// We will pass this function as argument to commit transaction function. +fn check_constraints(datastore: &mut D, committed: &Committed) -> Result +where + D: DataStore, +{ + // Get settings to commit from pending transaction + let settings_to_commit = datastore + .get_prefix("settings.", committed) + .context(error::DataStoreSnafu { op: "get_prefix" })?; + + // Get metadata from pending transaction + let mut transaction_metadata = datastore + .get_metadata_prefix("settings.", committed, &None as &Option<&str>) + .context(error::DataStoreSnafu { + op: "get_metadata_prefix", + })?; + + // Vector(metadata_key, key, value) + let mut metadata_to_commit: Vec<(Key, Key, String)> = Vec::new(); + + // Parse and validate all the metadata enteries from pending transaction + for (key, value) in transaction_metadata.iter_mut() { + for (metadata_key, metadata_value) in value { + // For now we are only processing the strength metadata from pending + // transaction to live + if metadata_key.name() != "strength" { + continue; + } + + // strength in pending transaction + let pending_strength: String = + deserialize_scalar::<_, ScalarError>(&metadata_value.clone()) + .with_context(|_| error::DeSerializeSnafu {})?; + + let pending_strength: Strength = + pending_strength + .parse::() + .context(error::ParseSnafu { + strength: pending_strength, + })?; + + // Get the setting strength in live + // get_metadata function returns Ok(None) in case strength does not exist + // We will consider this case as strength equals strong. + let committed_strength: Strength = datastore + .get_metadata(metadata_key, key, &Committed::Live) + .context(error::DataStoreSnafu { op: "get_metadata" })? + .map(|x| x.parse::()) + .transpose() + .context(error::TransposeSnafu)? + .unwrap_or_default(); + + // The get key funtion returns Ok(None) in case if the path does not exist + // and error if some path exist and some error occurred in fetching + // Hence we we will return error in case of error + // from get key function and continue to add/change to weak key + // if the value is None. + let value = datastore + .get_key(key, &Committed::Live) + .context(error::DataStoreSnafu { op: "get_key" })?; + + trace!( + "check_constraints: key: {:?}, metadata_key: {:?}, metadata_value: {:?}", + key.name(), + metadata_key.name(), + metadata_value + ); + + match (pending_strength, committed_strength) { + (Strength::Weak, Strength::Strong) => { + // Do not change from strong to weak if setting exists + // otherwise commit strength metadata with value as "weak" + if value.is_some() { + return Ok(ConstraintCheckResult::Reject( + "Cannot change setting strength from strong to weak".to_string(), + )); + } else { + let met_value = serialize_scalar::<_, ScalarError>(&pending_strength) + .with_context(|_| error::SerializeSnafu {})?; + + metadata_to_commit.push((metadata_key.clone(), key.clone(), met_value)); + } + } + (Strength::Strong, Strength::Weak) => { + let met_value = serialize_scalar::<_, ScalarError>(&pending_strength) + .with_context(|_| error::SerializeSnafu {})?; + metadata_to_commit.push((metadata_key.clone(), key.clone(), met_value)); + } + (Strength::Weak, Strength::Weak) => { + trace!("The strength for setting {} is already weak", key.name()); + continue; + } + (Strength::Strong, Strength::Strong) => { + trace!("The strength for setting {} is already strong", key.name()); + continue; + } + }; + } + } + + let approved_write = ApprovedWrite { + settings: settings_to_commit, + metadata: metadata_to_commit, + }; + + Ok(ConstraintCheckResult::from(Some(approved_write))) +} + +fn check_constraints_wrapper( + datastore: &mut D, + committed: &Committed, +) -> Result> +where + D: DataStore, +{ + check_constraints::(datastore, committed) + .map_err(|e| Box::new(e) as Box) +} + /// Makes live any pending settings in the datastore, returning the changed keys. pub(crate) fn commit_transaction(datastore: &mut D, transaction: &str) -> Result> where D: DataStore, { datastore - .commit_transaction(transaction) + .commit_transaction(transaction, &check_constraints_wrapper::) .context(error::DataStoreSnafu { op: "commit" }) } @@ -786,7 +1048,7 @@ mod test { let mut ds = MemoryDataStore::new(); let tx = "test transaction"; let pending = Committed::Pending { tx: tx.into() }; - set_settings(&mut ds, &settings, tx).unwrap(); + set_settings(&mut ds, &settings, tx, Strength::Strong).unwrap(); // Retrieve directly let key = Key::new(KeyType::Data, "settings.motd").unwrap(); @@ -805,6 +1067,7 @@ mod test { &Key::new(KeyType::Meta, "my-meta").unwrap(), &Key::new(KeyType::Data, data_key).unwrap(), "\"json string\"", + &Committed::Live, ) .unwrap(); } @@ -829,6 +1092,7 @@ mod test { &Key::new(KeyType::Meta, "my-meta").unwrap(), &Key::new(KeyType::Data, data_key).unwrap(), "\"json string\"", + &Committed::Live, ) .unwrap(); } @@ -863,9 +1127,9 @@ mod test { get_settings(&ds, &Committed::Live).unwrap_err(); // Commit, pending -> live - commit_transaction(&mut ds, tx).unwrap(); + commit_transaction::(&mut ds, tx).unwrap(); - // No more pending settings + // // No more pending settings get_settings(&ds, &pending).unwrap_err(); // Confirm live let settings = get_settings(&ds, &Committed::Live).unwrap(); diff --git a/sources/api/apiserver/src/server/error.rs b/sources/api/apiserver/src/server/error.rs index a5a3363e1..8145a7c52 100644 --- a/sources/api/apiserver/src/server/error.rs +++ b/sources/api/apiserver/src/server/error.rs @@ -92,6 +92,9 @@ pub enum Error { source: datastore::deserialization::Error, }, + #[snafu(display("Unable to deserialize data: {}", source))] + DeSerialize { source: serde_json::Error }, + #[snafu(display("Unable to serialize data: {}", source))] Serialize { source: serde_json::Error }, @@ -240,9 +243,37 @@ pub enum Error { stdout: Vec, source: serde_json::Error, }, + + #[snafu(display("Error deserializing response value to SettingsGenerator: {}", source))] + DeserializeSettingsGenerator { source: serde_json::Error }, + + #[snafu(display( + "Provided strength is not one of weak or strong. The given strength is: {}. {}", + strength, + source + ))] + InvalidStrength { + strength: String, + source: serde_plain::Error, + }, + + #[snafu(display( + "Trying to change the strength from strong to weak for key: {}, Operation restricted", + key + ))] + DisallowStrongToWeakStrength { key: String }, + + #[snafu(display("Unable to transpose the strength result. Error: {} ", source))] + Transpose { source: serde_plain::Error }, + + #[snafu(display("Unable to parse the given strength: {}. Error: ", strength))] + Parse { + strength: String, + source: serde_plain::Error, + }, } -pub type Result = std::result::Result; +pub type Result = std::result::Result; impl From for actix_web::HttpResponse { fn from(e: Error) -> Self { diff --git a/sources/api/apiserver/src/server/mod.rs b/sources/api/apiserver/src/server/mod.rs index 36667d36d..cf009fa4c 100644 --- a/sources/api/apiserver/src/server/mod.rs +++ b/sources/api/apiserver/src/server/mod.rs @@ -11,13 +11,14 @@ pub use error::Error; use actix_web::{ body::BoxBody, error::ResponseError, web, App, HttpRequest, HttpResponse, HttpServer, Responder, }; +use core::str; use datastore::{serialize_scalar, Committed, FilesystemDataStore, Key, KeyType, Value}; use error::Result; use fs2::FileExt; use http::StatusCode; use log::info; use model::ephemeral_storage::{Bind, Init}; -use model::{ConfigurationFiles, Model, Report, Services, Settings}; +use model::{ConfigurationFiles, Model, Report, Services, Settings, SettingsGenerator, Strength}; use nix::unistd::{chown, Gid}; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt}; @@ -28,6 +29,7 @@ use std::os::unix::fs::PermissionsExt; use std::os::unix::process::ExitStatusExt; use std::path::{Path, PathBuf}; use std::process::Command; +use std::str::FromStr; use std::sync; use thar_be_updates::status::{UpdateStatus, UPDATE_LOCKFILE}; use tokio::process::Command as AsyncCommand; @@ -108,6 +110,14 @@ where web::post().to(commit_transaction_and_apply), ), ) + .service( + web::scope("/v2") + .route("/tx", web::get().to(get_transaction_v2)) + .route( + "/metadata/setting-generators", + web::get().to(get_setting_generators_v2), + ), + ) .service(web::scope("/os").route("", web::get().to(get_os_info))) .service( web::scope("/metadata") @@ -304,8 +314,9 @@ async fn patch_settings( data: web::Data, ) -> Result { let transaction = transaction_name(&query); + let strength = query_strength(&query)?; let mut datastore = data.ds.write().ok().context(error::DataStoreLockSnafu)?; - controller::set_settings(&mut *datastore, &settings, transaction)?; + controller::set_settings(&mut *datastore, &settings, transaction, strength)?; Ok(HttpResponse::NoContent().finish()) // 204 } @@ -318,13 +329,14 @@ async fn patch_settings_key_pair( // Convert to a Map of Key Value pairs. let settings_key_pair_map = construct_key_pair_map(&settings.request_payload)?; let transaction = transaction_name(&query); + let strength = query_strength(&query)?; let mut datastore = data.ds.write().ok().context(error::DataStoreLockSnafu)?; // We massage the values in the input key pair map. // The data store deserialization code understands how to turn the key names // (a.b.c) and serialized values into the nested Settings structure. let settings_model = datastore::deserialization::from_map(&settings_key_pair_map) .context(error::DeserializeMapSnafu)?; - controller::set_settings(&mut *datastore, &settings_model, transaction)?; + controller::set_settings(&mut *datastore, &settings_model, transaction, strength)?; Ok(HttpResponse::NoContent().finish()) // 204 } @@ -342,9 +354,29 @@ async fn get_transaction( let transaction = transaction_name(&query); let datastore = data.ds.read().ok().context(error::DataStoreLockSnafu)?; let data = controller::get_transaction(&*datastore, transaction)?; + Ok(SettingsResponse(data)) } +/// Get any pending settings in the given transaction, or the "default" transaction if unspecified. +async fn get_transaction_v2( + query: web::Query>, + data: web::Data, +) -> Result { + let transaction = transaction_name(&query); + let datastore = data.ds.read().ok().context(error::DataStoreLockSnafu)?; + let settings = controller::get_transaction(&*datastore, transaction)?; + let transaction_metadata = + controller::get_transaction_metadata(&*datastore, transaction, None)?; + + let data = SettingsWithMetadata { + settings, + metadata: transaction_metadata.inner, + }; + + Ok(SettingsResponseWithMetadata(data)) +} + /// Delete the given transaction, or the "default" transaction if unspecified. async fn delete_transaction( query: web::Query>, @@ -365,7 +397,10 @@ async fn commit_transaction( let transaction = transaction_name(&query); let mut datastore = data.ds.write().ok().context(error::DataStoreLockSnafu)?; - let changes = controller::commit_transaction(&mut *datastore, transaction)?; + let changes = controller::commit_transaction::( + &mut *datastore, + transaction, + )?; if changes.is_empty() { return error::CommitWithNoPendingSnafu.fail(); @@ -397,7 +432,10 @@ async fn commit_transaction_and_apply( let transaction = transaction_name(&query); let mut datastore = data.ds.write().ok().context(error::DataStoreLockSnafu)?; - let changes = controller::commit_transaction(&mut *datastore, transaction)?; + let changes = controller::commit_transaction::( + &mut *datastore, + transaction, + )?; if changes.is_empty() { return error::CommitWithNoPendingSnafu.fail(); @@ -453,6 +491,43 @@ async fn get_affected_services( /// Get all settings that have setting-generator metadata async fn get_setting_generators(data: web::Data) -> Result { + let datastore = data.ds.read().ok().context(error::DataStoreLockSnafu)?; + let metadata_for_keys = + controller::get_metadata_for_all_data_keys(&*datastore, "setting-generator")?; + let mut resp: HashMap = HashMap::new(); + + for (key, value) in metadata_for_keys.iter() { + match value { + Value::String(command) => { + resp.insert( + key.to_string(), + serde_json::Value::String(command.to_string()), + ); + } + Value::Object(obj) => { + let setting_generator: SettingsGenerator = + serde_json::from_value(Value::Object(obj.clone())) + .context(error::DeserializeSettingsGeneratorSnafu)?; + // Not return weak setting generators because the customers using + // v1 of this api are not aware of the strength. + if setting_generator.strength == Strength::Strong { + resp.insert( + key.to_string(), + serde_json::Value::String(setting_generator.command), + ); + } + } + // We know it is not possible, as in default we set the setting-generator + // as string or object + _ => { + error!("Invalid value type for key '{}': {:?}", key, value); + } + } + } + Ok(MetadataResponse(resp)) +} + +async fn get_setting_generators_v2(data: web::Data) -> Result { let datastore = data.ds.read().ok().context(error::DataStoreLockSnafu)?; let resp = controller::get_metadata_for_all_data_keys(&*datastore, "setting-generator")?; Ok(MetadataResponse(resp)) @@ -753,8 +828,19 @@ fn transaction_name(query: &web::Query>) -> &str { query.get("tx").map(String::as_str).unwrap_or("default") } -// Helpers methods for the 'set' API +fn query_strength(query: &web::Query>) -> Result { + if let Some(strength) = query.get("strength") { + Ok( + Strength::from_str(strength).context(error::InvalidStrengthSnafu { + strength: strength.to_string(), + })?, + ) + } else { + Ok(Strength::default()) + } +} +// Helpers methods for the 'set' API fn construct_key_pair_map(settings_key_pair_vec: &Vec) -> Result> { let mut settings_key_pair_map = HashMap::new(); for settings_key_pair in settings_key_pair_vec { @@ -892,6 +978,12 @@ impl ResponseError for error::Error { UpdateLockOpen { .. } => StatusCode::INTERNAL_SERVER_ERROR, ReportExec { .. } => StatusCode::INTERNAL_SERVER_ERROR, ReportResult { .. } => StatusCode::INTERNAL_SERVER_ERROR, + DeserializeSettingsGenerator { .. } => StatusCode::BAD_REQUEST, + DeSerialize { .. } => StatusCode::BAD_REQUEST, + InvalidStrength { .. } => StatusCode::BAD_REQUEST, + DisallowStrongToWeakStrength { .. } => StatusCode::BAD_REQUEST, + Transpose { .. } => StatusCode::BAD_REQUEST, + Parse { .. } => StatusCode::BAD_REQUEST, }; HttpResponse::build(status_code).body(self.to_string()) @@ -944,6 +1036,16 @@ macro_rules! impl_responder_for { struct ModelResponse(serde_json::Value); impl_responder_for!(ModelResponse, self, self.0); +/// This lets us respond from our handler methods with a Settings (or Result) +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +struct SettingsWithMetadata { + settings: Settings, + metadata: HashMap>, +} + +struct SettingsResponseWithMetadata(SettingsWithMetadata); +impl_responder_for!(SettingsResponseWithMetadata, self, self.0); + /// This lets us respond from our handler methods with a Settings (or Result) struct SettingsResponse(Settings); impl_responder_for!(SettingsResponse, self, self.0);