[sync] unified configuration for validator (#191)

Co-authored-by: zoz <[email protected]> Co-authored-by: 0o-de-lally <[email protected]> Co-authored-by: Kalvis Kuskis <[email protected]> Co-authored-by: Ubuntu <[email protected]>
0LNetworkCommunity · Feb 29, 2024 · 013d687 · 013d687
1 parent f742242
commit 013d687
Show file tree

Hide file tree

Showing 17 changed files with 388 additions and 97 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/tools/config/Cargo.toml b/tools/config/Cargo.toml
@@ -24,6 +24,7 @@ libra-wallet = { workspace = true }
 reqwest = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
+serde = { workspace = true }
 tokio = { workspace = true }
 url = { workspace = true }
 

diff --git a/tools/config/src/config_cli.rs b/tools/config/src/config_cli.rs
@@ -1,4 +1,6 @@
-use crate::make_yaml_public_fullnode::{download_genesis, init_fullnode_yaml};
+use crate::make_yaml_public_fullnode::{
+    download_genesis, get_genesis_waypoint, init_fullnode_yaml,
+};
 use crate::validator_config::{validator_dialogue, vfn_dialogue};
 use crate::{legacy_config, make_profile};
 use anyhow::{Context, Result};
@@ -240,12 +242,14 @@ impl ConfigCli {
                     );
                     std::fs::create_dir_all(&data_path)?;
                 }
+                download_genesis(Some(data_path.clone()), None).await?;
+                let _ = get_genesis_waypoint(Some(data_path.clone())).await?;
                 validator_dialogue(&data_path, None).await?;
                 println!("Validators' config initialized.");
                 Ok(())
             }
             Some(ConfigSub::FullnodeInit { home_path }) => {
-                download_genesis(home_path.to_owned()).await?;
+                download_genesis(home_path.to_owned(), None).await?;
                 println!("downloaded genesis block");
 
                 let p = init_fullnode_yaml(home_path.to_owned(), true).await?;

diff --git a/tools/config/src/make_yaml_public_fullnode.rs b/tools/config/src/make_yaml_public_fullnode.rs
@@ -6,13 +6,25 @@ use diem_types::{
     PeerId,
 };
 use libra_types::global_config_dir;
+use serde::Deserialize;
 use std::{
     collections::HashMap,
     path::{Path, PathBuf},
 };
 
 const FN_FILENAME: &str = "fullnode.yaml";
 const VFN_FILENAME: &str = "vfn.yaml";
+const DEFAULT_WAYPOINT_VERSION: &str = "6.9.0";
+#[derive(Debug, Deserialize)]
+#[allow(dead_code)]
+struct GithubContent {
+    name: String,
+    path: String,
+    #[serde(rename = "type")]
+    content_type: String, // Use `content_type` to avoid conflict with Rust's `type` keyword
+    download_url: Option<String>,
+}
+
 /// fetch seed peers and make a yaml file from template
 pub async fn init_fullnode_yaml(
     home_dir: Option<PathBuf>,
@@ -52,7 +64,6 @@ pub fn add_peers_to_yaml(
 
     Ok(())
 }
-
 /// get seed peers from an upstream url
 pub async fn fetch_seed_addresses(
     url: Option<&str>,
@@ -161,39 +172,51 @@ mempool:
 }
 
 /// download genesis blob
-pub async fn download_genesis(home_dir: Option<PathBuf>) -> anyhow::Result<()> {
-    let bytes = reqwest::get(
-        "https://raw.githubusercontent.com/0LNetworkCommunity/epoch-archive-mainnet/main/upgrades/v6.9.0/genesis.blob",
-    )
-    .await?
-    .bytes()
-    .await?;
-
+pub async fn download_genesis(
+    home_dir: Option<PathBuf>,
+    genesis_path: Option<&str>,
+) -> anyhow::Result<()> {
+    let path = genesis_path.unwrap_or("https://raw.githubusercontent.com/0LNetworkCommunity/epoch-archive-mainnet/main/upgrades/v6.9.0/genesis.blob");
+    let bytes = reqwest::get(path).await?.bytes().await?;
     let home = home_dir.unwrap_or_else(global_config_dir);
     let genesis_dir = home.join("genesis/");
     std::fs::create_dir_all(&genesis_dir)?;
     let p = genesis_dir.join("genesis.blob");
-
     std::fs::write(p, bytes)?;
-
     Ok(())
 }
 
 pub async fn get_genesis_waypoint(home_dir: Option<PathBuf>) -> anyhow::Result<Waypoint> {
-    let wp_string = reqwest::get(
-        "https://raw.githubusercontent.com/0LNetworkCommunity/epoch-archive-mainnet/main/upgrades/v6.9.0/waypoint.txt",
-    )
-    .await?
-    .text()
-    .await?;
-
+    // Base URL for GitHub API requests
+    let base_url =
+        "https://api.github.com/repos/0LNetworkCommunity/epoch-archive-mainnet/contents/upgrades";
+    let client = reqwest::Client::new();
+    let resp = client
+        .get(base_url)
+        .header("User-Agent", "request")
+        .send()
+        .await?
+        .json::<Vec<GithubContent>>()
+        .await?;
+    // Find the latest version by parsing version numbers and sorting
+    let latest_version = resp
+        .iter()
+        .filter_map(|entry| entry.name.split('v').nth(1)) // Assuming the name is 'vX.X.X'
+        .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+
+    let latest_path = format!(
+        "{}/v{}/waypoint.txt",
+        "https://raw.githubusercontent.com/0LNetworkCommunity/epoch-archive-mainnet/main/upgrades",
+        latest_version.unwrap_or(DEFAULT_WAYPOINT_VERSION)
+    );
+    // Fetch the latest waypoint
+    let wp_string = reqwest::get(&latest_path).await?.text().await?;
     let home = home_dir.unwrap_or_else(libra_types::global_config_dir);
     let genesis_dir = home.join("genesis/");
     let p = genesis_dir.join("waypoint.txt");
 
     std::fs::write(p, &wp_string)?;
-
-    wp_string.parse()
+    wp_string.trim().parse::<Waypoint>()
 }
 
 #[tokio::test]
@@ -241,7 +264,7 @@ async fn persist_genesis() {
 
     let path = p.path().to_owned();
 
-    download_genesis(Some(path)).await.unwrap();
+    download_genesis(Some(path), None).await.unwrap();
     let l = std::fs::read_dir(p.path())
         .unwrap()
         .next()

diff --git a/tools/config/src/make_yaml_validator.rs b/tools/config/src/make_yaml_validator.rs
@@ -11,7 +11,7 @@ pub const NODE_YAML_FILE: &str = "validator.yaml";
 
 /// Create a validator yaml file to start validator node.
 /// NOTE: this will not work for fullnodes
-pub fn save_validator_yaml(home_dir: Option<PathBuf>) -> Result<PathBuf> {
+pub async fn save_validator_yaml(home_dir: Option<PathBuf>) -> Result<PathBuf> {
     let home_dir = home_dir.unwrap_or_else(global_config_dir);
     let path = home_dir.display().to_string();
 
@@ -55,6 +55,8 @@ full_node_networks:
   identity:
     type: 'from_file'
     path: {path}/validator-identity.yaml
+- network_id: 'public'
+  listen_address: '/ip4/0.0.0.0/tcp/6182'
 
 api:
   enabled: true
@@ -66,20 +68,23 @@ api:
 
     write_to_user_only_file(&output_file, NODE_YAML_FILE, template.as_bytes())?;
 
+    let peers = crate::make_yaml_public_fullnode::fetch_seed_addresses(None).await?;
+    crate::make_yaml_public_fullnode::add_peers_to_yaml(&output_file, peers)?;
+
     Ok(output_file)
 }
 
-#[test]
+#[tokio::test]
 #[ignore] // TODO: not sure why this parsing is failing, when node can start with this file.
-fn test_yaml() {
+async fn test_yaml() {
     use diem_config::config::NodeConfig;
     use libra_wallet::utils::from_yaml;
 
     let path = global_config_dir().join("test_yaml");
 
     std::fs::create_dir_all(&path).unwrap();
 
-    let file = save_validator_yaml(Some(path.clone())).unwrap();
+    let file = save_validator_yaml(Some(path.clone())).await.unwrap();
 
     let read = std::fs::read_to_string(file).unwrap();
 

diff --git a/tools/config/src/validator_config.rs b/tools/config/src/validator_config.rs
@@ -15,7 +15,7 @@ use libra_wallet::validator_files::SetValidatorConfiguration;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
-pub fn initialize_validator(
+pub async fn initialize_validator(
     home_path: Option<PathBuf>,
     username: Option<&str>,
     host: HostAndPort,
@@ -33,7 +33,7 @@ pub fn initialize_validator(
         .set_config_files()?;
     OLProgress::complete("saved validator registration files locally");
 
-    make_yaml_validator::save_validator_yaml(home_path.clone())?;
+    make_yaml_validator::save_validator_yaml(home_path.clone()).await?;
     OLProgress::complete("saved validator node yaml file locally");
 
     // TODO: nice to have
@@ -111,7 +111,8 @@ pub async fn validator_dialogue(
             None,
             keep_legacy_address,
             None,
-        )?;
+        )
+        .await?;
 
         // now set up the vfn.yaml on the same host for convenience
         vfn_dialogue(
@@ -166,8 +167,8 @@ pub async fn vfn_dialogue(
     Ok(())
 }
 
-#[test]
-fn test_validator_files_config() {
+#[tokio::test]
+async fn test_validator_files_config() {
     use libra_types::global_config_dir;
     let alice_mnem = "talent sunset lizard pill fame nuclear spy noodle basket okay critic grow sleep legend hurry pitch blanket clerk impose rough degree sock insane purse".to_string();
     let h = HostAndPort::local(6180).unwrap();
@@ -184,6 +185,7 @@ fn test_validator_files_config() {
         false,
         None,
     )
+    .await
     .unwrap();
 
     std::fs::remove_dir_all(&test_path).unwrap();

diff --git a/tools/genesis/src/main.rs b/tools/genesis/src/main.rs
@@ -152,14 +152,17 @@ async fn main() -> anyhow::Result<()> {
             ip_list,
             supply_settings,
             json_legacy,
-        }) => testnet_setup::setup(
-            &me,
-            &ip_list,
-            cli.chain.unwrap_or(NamedChain::TESTING),
-            cli.home_dir.unwrap_or_else(global_config_dir),
-            &supply_settings,
-            json_legacy,
-        )?,
+        }) => {
+            testnet_setup::setup(
+                &me,
+                &ip_list,
+                cli.chain.unwrap_or(NamedChain::TESTING),
+                cli.home_dir.unwrap_or_else(global_config_dir),
+                &supply_settings,
+                json_legacy,
+            )
+            .await?
+        }
         _ => {
             println!("\nIf you're looking for trouble \nYou came to the right place");
         }

diff --git a/tools/genesis/src/testnet_setup.rs b/tools/genesis/src/testnet_setup.rs
@@ -4,7 +4,7 @@ use libra_config::validator_config;
 use libra_types::{exports::NamedChain, legacy_types::fixtures::TestPersona};
 use std::{fs, net::Ipv4Addr, path::PathBuf, thread, time};
 
-pub fn setup(
+pub async fn setup(
     me: &TestPersona,
     ip_list: &[Ipv4Addr],
     chain: NamedChain,
@@ -40,7 +40,8 @@ pub fn setup(
         Some(me.get_persona_mnem()),
         false,
         Some(chain),
-    )?;
+    )
+    .await?;
 
     // create validator configurations from fixtures
     // without needing to use a github repo to register and read

diff --git a/tools/query/Cargo.toml b/tools/query/Cargo.toml
@@ -14,9 +14,12 @@ anyhow = { workspace = true }
 clap = { workspace = true }
 diem-debugger = { workspace = true }
 diem-sdk = { workspace = true }
+diem-types = { workspace = true }
+diem-api-types = { workspace = true }
 indoc = { workspace = true }
 libra-types = { workspace = true }
 serde_json = { workspace = true }
+serde = { workspace = true }
 tokio = { workspace = true }
 
 [dev-dependencies]