diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 91d0102..472120e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,8 @@ name: Continuous Integration (CI) -on: [push] +on: + push: + branches: + - main jobs: test: diff --git a/Cargo.toml b/Cargo.toml index 822aea3..7a6120a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,34 +1,31 @@ [package] name = "freighter" -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "MIT" homepage = "" repository = "https://github.com/open-rust-Initiative/freighter" documentation = "" -readme = "README.md" -description = """ - -""" +description = "" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.66" +anyhow = "1.0.71" thiserror = "1.0.36" -clap = "4.3.1" -git2 = "0.17.2" -git2-curl = "0.18" +clap = "4.3.4" +git2 = "0.18.1" +git2-curl = "0.19" url = "2.3.1" -serde = { version = "1.0.159", features = ["derive"] } -serde_json = "1.0.96" +serde = { version = "1.0.164", features = ["derive"] } +serde_json = "1.0.97" walkdir = "2.3.2" reqwest = { version = "0.11.13", features = ["blocking"] } openssl = { version = "0.10.54", features = ["vendored"] } chrono = "0.4.26" -sha2 = "0.10.6" +sha2 = "0.10.7" dirs = "5.0.0" -toml = "0.7.1" +toml = "0.8.8" log4rs = {version = "1.2.0", features = ["toml_format"] } tokio = { version = "1.28.2", features = ["full"] } warp = { version = "0.3.3", features = ["tls"] } diff --git a/README.md b/README.md index 3c4c26b..380e746 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,39 @@ When developing a program using Rust in a company, we need to host a proxy for c ### How to use the Freighter? +Freighter's functionality mainly consists of four parts: synchronizing crates index and crates; syncing the rustup-init files; syncing the rust toolchain files; providing a HTTP server that support static file server, parse the Git protocol, and offering API services such as crate publication. + +Freighter can be executed as a standalone executable program. You can build it using the **cargo build --release** command and then copy it to your `/usr/local/bin directory`. + +#### 1. Synchronizing Crates Index and Crates +To sync crate files, Freighter needs to first sync the crates index. You can use the following command to sync the index file: + +```bash +freighter crates pull +``` + +This command will create a crates.io-index directory in the default path **/Users/${USERNAME}/freighter** and fetch the index. If the index already exists, it will attempt to update it. +You can also use the **-c** parameter to specify a **working directory** to change the storage location of the index and crates: + +```bash +freighter -c /path/to/wokring_dir crates pull +``` + +**Full download**: Next, you can use the download command with the init parameter to download the full set of crates files: + +```bash +freighter -c /path/to/wokring_dir crates download --init +``` + +**Incremental update**: Without the init parameter, Freighter will compare log records in the **working directory** to determine the index and crates that need incremental updates: + +```bash +freighter -c /path/to/wokring_dir crates download +``` + +#### 2.Syncing the rustup-init Files +#### 3.Syncing the Rust Toolchain Files +#### 4.Http Server ### How to contribute? diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 0785c26..a29bc35 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -25,7 +25,12 @@ pub mod server; /// point and the `exec` function is logic entry. /// pub fn builtin() -> Vec { - vec![crates::cli(), rustup_init::cli(), channel::cli(), server::cli()] + vec![ + crates::cli(), + rustup_init::cli(), + channel::cli(), + server::cli(), + ] } /// diff --git a/src/commands/server.rs b/src/commands/server.rs index 5c3de93..c9516ef 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -32,9 +32,15 @@ use crate::server::file_server::{self, FileServer}; pub fn cli() -> clap::Command { clap::Command::new("server") - .arg(arg!(-i --"ip" "spcify the ip address").value_parser(value_parser!(IpAddr)).default_value("127.0.0.1")) .arg( - arg!(-p --"port" "specify the listening port").value_parser(value_parser!(u16)).default_value("8000"), + arg!(-i --"ip" "specify the ip address") + .value_parser(value_parser!(IpAddr)) + .default_value("127.0.0.1"), + ) + .arg( + arg!(-p --"port" "specify the listening port") + .value_parser(value_parser!(u16)) + .default_value("8000"), ) .arg( arg!(-c --"cert-path" "Path to a TLS certificate file") diff --git a/src/download.rs b/src/download.rs index 26b71fb..ad73d74 100644 --- a/src/download.rs +++ b/src/download.rs @@ -123,7 +123,7 @@ pub fn download_and_check_hash( let hex = format!("{:x}", result); //if need to calculate hash - if let Some(..) = check_sum { + if check_sum.is_some() { return if hex == check_sum.unwrap() { tracing::info!("###[ALREADY] \t{:?}", file); Ok(false) @@ -145,7 +145,7 @@ pub fn download_and_check_hash( pub fn encode_huaweicloud_url(url: &mut Url) { if let Some(domain) = url.domain() { - if domain.contains("myhuaweicloud.com") && url.path().starts_with("/crates"){ + if domain.contains("myhuaweicloud.com") && url.path().starts_with("/crates") { let mut path = PathBuf::from(url.path()); let encode_path: String = byte_serialize(path.file_name().unwrap().to_str().unwrap().as_bytes()).collect(); diff --git a/src/handler/crates_file.rs b/src/handler/crates_file.rs index 3af49a0..cbe88cf 100644 --- a/src/handler/crates_file.rs +++ b/src/handler/crates_file.rs @@ -28,7 +28,7 @@ use crate::errors::FreightResult; use crate::handler::index; use super::index::CrateIndex; -use super::DownloadMode; +use super::{utils, DownloadMode}; /// CratesOptions preserve the sync subcommand config #[derive(Clone, Debug)] @@ -84,22 +84,19 @@ impl Default for CratesOptions { impl CratesOptions { // the path rules of craes index file pub fn get_index_path(&self, name: &str) -> PathBuf { - let suffix = match name.len() { - 1..=2 => format!("{}/{}", name.len(), name), - 3 => format!("{}/{}/{}", name.len(), &name[0..1], name), - _ => format!("{}/{}/{}", &name[0..2], &name[2..4], name), - }; + let suffix = utils::index_suffix(name); self.index.path.join(suffix) } } /// Crate preserve the crates info parse from registry json file #[derive(Serialize, Deserialize, Debug)] -pub struct Crate { +pub struct IndexFile { pub name: String, pub vers: String, pub deps: Vec, - pub cksum: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub cksum: Option, pub features: BTreeMap>, #[serde(skip_serializing_if = "Option::is_none")] pub features2: Option>>, @@ -316,14 +313,15 @@ pub fn parse_index_and_download( for line in buffered.lines() { let line = line.unwrap(); - let c: Crate = serde_json::from_str(&line).unwrap(); + let c: IndexFile = serde_json::from_str(&line).unwrap(); let err_record = Arc::clone(err_record); let opts = opts.clone(); let url = Url::parse(&format!( "{}/{}/{}-{}.crate", opts.config.domain, &c.name, &c.name, &c.vers - )).unwrap(); + )) + .unwrap(); let file = opts .crates_path @@ -352,7 +350,7 @@ pub fn download_crates_with_log( path: PathBuf, opts: &CratesOptions, url: Url, - c: Crate, + index_file: IndexFile, err_record: Arc>, ) -> FreightResult { let down_opts = &DownloadOptions { @@ -361,7 +359,7 @@ pub fn download_crates_with_log( path, }; - match download_and_check_hash(down_opts, Some(&c.cksum), false) { + match download_and_check_hash(down_opts, Some(&index_file.cksum.unwrap()), false) { Ok(download_succ) => { let path = &down_opts.path; if download_succ && opts.upload { @@ -383,8 +381,8 @@ pub fn download_crates_with_log( Err(err) => { let mut err_file = err_record.lock().unwrap(); let err_crate = ErrorCrate { - name: c.name, - vers: c.vers, + name: index_file.name, + vers: index_file.vers, time: Utc::now().timestamp().to_string(), }; let json = serde_json::to_string(&err_crate).unwrap(); diff --git a/src/handler/mod.rs b/src/handler/mod.rs index 34bfb85..3be0061 100644 --- a/src/handler/mod.rs +++ b/src/handler/mod.rs @@ -29,3 +29,15 @@ impl DownloadMode { } } } + +pub mod utils { + + // the path rules of crates index file + pub fn index_suffix(name: &str) -> String { + match name.len() { + 1..=2 => format!("{}/{}", name.len(), name), + 3 => format!("{}/{}/{}", name.len(), &name[0..1], name), + _ => format!("{}/{}/{}", &name[0..2], &name[2..4], name), + } + } +} diff --git a/src/handler/rustup.rs b/src/handler/rustup.rs index 2fe7905..4d95d64 100644 --- a/src/handler/rustup.rs +++ b/src/handler/rustup.rs @@ -6,8 +6,8 @@ //! use rayon::{ThreadPool, ThreadPoolBuilder}; -use url::Url; use std::{path::PathBuf, sync::Arc}; +use url::Url; use crate::{ config::ProxyConfig, diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 4c13f9d..9cbca0a 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -32,6 +32,7 @@ pub struct FileServer { #[tokio::main] pub async fn start(config: &Config, file_server: &FileServer) { tracing_subscriber::fmt::init(); + // storage::init().await; let routes = filters::build_route(config.to_owned()) .recover(handlers::handle_rejection) .with(warp::trace::request()); @@ -68,9 +69,17 @@ pub async fn start(config: &Config, file_server: &FileServer) { mod filters { use std::path::PathBuf; + use bytes::{Buf, Bytes}; use warp::{Filter, Rejection}; - use crate::{config::Config, server::git_protocol::GitCommand}; + use crate::{ + config::Config, + server::{ + file_server::utils, + git_protocol::GitCommand, + model::{CratesPublish, Errors, PublishRsp}, + }, + }; use super::handlers; @@ -86,8 +95,64 @@ mod filters { // GET /dist/... => ./dist/.. dist(config.clone()) .or(rustup(config.clone())) - .or(crates(config)) + .or(crates(config.clone())) .or(git(git_work_dir)) + .or(publish(config.clone())) + .or(sparse_index(config)) + } + + pub fn publish( + config: Config, + ) -> impl Filter + Clone { + warp::path!("api" / "v1" / "crates" / "new") + .and(warp::body::bytes()) + .and(with_config(config)) + .map(|mut body: Bytes, config: Config| { + let json_len = utils::get_usize_from_bytes(body.copy_to_bytes(4)); + + tracing::info!("json_len: {:?}", json_len); + let json = body.copy_to_bytes(json_len); + tracing::info!("raw json: {:?}", json); + + let parse_result = serde_json::from_slice::(json.as_ref()); + let crate_len = utils::get_usize_from_bytes(body.copy_to_bytes(4)); + let file_content = body.copy_to_bytes(crate_len); + + match parse_result { + Ok(result) => { + println!("JSON: {:?}", result); + let work_dir = config.work_dir.unwrap(); + utils::save_crate_index( + &result, + &file_content, + work_dir.join("crates.io-index"), + ); + utils::save_crate_file(&result, &file_content, work_dir.join("crates")); + // let std::fs::write(); + // 1.verify name and version from local db + // 2.call remote server to check info in crates.io + warp::reply::json(&PublishRsp::default()) + } + Err(err) => warp::reply::json(&Errors::new(err.to_string())), + } + }) + } + + pub fn sparse_index( + config: Config, + ) -> impl Filter + Clone { + warp::path("index") + .and(warp::path::tail()) + .and(with_config(config)) + .and_then(|tail: warp::path::Tail, config: Config| async move { + handlers::return_files( + config.rustup.serve_domains.unwrap(), + config.work_dir.unwrap(), + PathBuf::from("crates.io-index").join(tail.as_str()), + false, + ) + .await + }) } // build '/dist/*' route, this route handle rust toolchian files request @@ -370,3 +435,42 @@ mod handlers { Ok(()) } } + +mod utils { + use std::{fs, path::PathBuf}; + + use crate::{ + handler::{crates_file::IndexFile, utils}, + server::model::CratesPublish, + }; + use bytes::Bytes; + use sha2::{Digest, Sha256}; + + pub fn get_usize_from_bytes(bytes: Bytes) -> usize { + let mut fixed_array = [0u8; 8]; + fixed_array[..4].copy_from_slice(&bytes[..4]); + usize::from_le_bytes(fixed_array) + } + + pub fn save_crate_index(json: &CratesPublish, content: &Bytes, work_dir: PathBuf) { + let suffix = utils::index_suffix(&json.name); + let index_path = work_dir.join(suffix); + //convert publish json to index file + let mut index_file: IndexFile = + serde_json::from_str(&serde_json::to_string(&json).unwrap()).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(content); + index_file.cksum = Some(format!("{:x}", hasher.finalize())); + fs::write(index_path, serde_json::to_string(&index_file).unwrap()).unwrap(); + } + + pub fn save_crate_file(json: &CratesPublish, content: &Bytes, work_dir: PathBuf) { + let crates_dir = work_dir.join(&json.name); + if !crates_dir.exists() { + fs::create_dir_all(&crates_dir).unwrap(); + } + let crates_file = crates_dir.join(format!("{}-{}.crate", json.name, json.vers)); + fs::write(crates_file, content).unwrap(); + } +} diff --git a/src/server/mod.rs b/src/server/mod.rs index 230d22e..493669c 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -7,3 +7,4 @@ pub mod file_server; pub mod git_protocol; +mod model; diff --git a/src/server/model.rs b/src/server/model.rs new file mode 100644 index 0000000..2ead94c --- /dev/null +++ b/src/server/model.rs @@ -0,0 +1,126 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct CratesPublish { + // List of strings of the authors. + // May be empty. + pub authors: Vec, + // Optional object of "status" badges. Each value is an object of + // arbitrary string to string mappings. + // crates.io has special interpretation of the format of the badges. + pub badges: Badge, + // Array of strings of categories for the package. + pub categories: Vec, + // Array of direct dependencies of the package. + pub deps: Vec, + // Description field from the manifest. + // May be null. crates.io requires at least some content. + pub description: String, + // String of the URL to the website for this package's documentation. + // May be null. + pub documentation: String, + // Set of features defined for the package. + // Each feature maps to an array of features or dependencies it enables. + // Cargo does not impose limitations on feature names, but crates.io + // requires alphanumeric ASCII, `_` or `-` characters. + pub features: BTreeMap>, + // String of the URL to the website for this package's home page. + // May be null. + pub homepage: String, + // Array of strings of keywords for the package. + pub keywords: Vec, + // String of the license for the package. + // May be null. crates.io requires either `license` or `license_file` to be set. + pub license: String, + // String of a relative path to a license file in the crate. + // May be null. + pub license_file: Option, + // The `links` string value from the package's manifest, or null if not + // specified. This field is optional and defaults to null. + pub links: Option, + pub name: String, + // String of the content of the README file. + // May be null. + pub readme: String, + // String of a relative path to a README file in the crate. + // May be null. + pub readme_file: String, + // String of the URL to the website for the source repository of this package. + // May be null. + pub repository: String, + // The version of the package being published. + pub vers: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Dep { + // Boolean of whether or not default features are enabled. + pub default_features: bool, + // Array of features (as strings) enabled for this dependency. + pub features: Vec, + // The dependency kind. + // "dev", "build", or "normal". + pub kind: String, + // Name of the dependency. + // If the dependency is renamed from the original package name, + // this is the original name. The new package name is stored in + // the `explicit_name_in_toml` field. + pub name: String, + // Boolean of whether or not this is an optional dependency. + pub optional: bool, + // The URL of the index of the registry where this dependency is + // from as a string. If not specified or null, it is assumed the + // dependency is in the current registry. + pub registry: String, + // The target platform for the dependency. + // null if not a target dependency. + // Otherwise, a string such as "cfg(windows)". + pub target: Option, + // The semver requirement for this dependency. + pub version_req: String, + // If the dependency is renamed, this is a string of the new + // package name. If not specified or null, this dependency is not + // renamed. + pub explicit_name_in_toml: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Badge {} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct PublishRsp { + // Optional object of warnings to display to the user. + pub warnings: Warning, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Warning { + // Array of strings of categories that are invalid and ignored. + pub invalid_categories: Vec, + // Array of strings of badge names that are invalid and ignored. + pub invalid_badges: Vec, + // Array of strings of arbitrary warnings to display to the user. + pub other: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Errors { + // Array of errors to display to the user. + pub errors: Vec, +} + +impl Errors { + pub fn new(detail: String) -> Errors { + Errors { + errors: vec![ErrorDetail { detail }], + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ErrorDetail { + // The error message as a string. + pub detail: String, +}