migration to single monorepo with lib support

.gitignore

@@ -1,3 +1,5 @@
-/target
+.vscode/
 .DS_Store
+
+/target
 Cargo.lock

Cargo.toml

@@ -1,13 +1,6 @@
-[package]
-name = "car-utils"
-version = "0.2.0"
-edition = "2021"
-authors = ["Join.Gong<[email protected]>", "Derek Anderson<[email protected]>", "Zeeshan.Sarwar<[email protected]>"]
-description = "This crate is the ipfs car file reader writer and utils."
-license = "MIT/Apache-2.0"
-repository = "https://github.com/blocklessnetwork/car-utils"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-clap = { version = "4.3.19", features = ["derive"] }
-blockless-car = "0.1.5"
+[workspace]
+members = [
+    "bin/car-utils",
+    "crates/blockless-car",
+]
+default-members = ["bin/car-utils"]

bin/car-utils/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "car-utils"
+version = "0.2.0"
+edition = "2021"
+authors = ["Join.Gong<[email protected]>", "Derek Anderson<[email protected]>", "Zeeshan.Sarwar<[email protected]>"]
+description = "This crate is the ipfs car file reader writer and utils CLI."
+license = "MIT/Apache-2.0"
+publish = false
+
+[dependencies]
+clap = { version = "4.3.19", features = ["derive"] }
+blockless-car = { path = "../../crates/blockless-car", default-features = false }

crates/blockless-car/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "blockless-car"
+version = "0.2.0"
+edition = "2021"
+authors = ["Join.Gong<[email protected]>", "Derek Anderson<[email protected]>", "Zeeshan.Sarwar<[email protected]>"]
+description = "This crate is the ipfs car file reader writer and utils library."
+license = "MIT/Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+ipld = { version = "0.15.0", package = "libipld"}
+ipld-cbor = {version = "0.15.0", package = "libipld-cbor"}
+thiserror = "1"
+cid = "0.9"
+integer-encoding = "3.0.4"
+path-absolutize = "3"
+quick-protobuf = { default-features = false, features = ["std"], version = "0.8" }

crates/blockless-car/README.md

@@ -0,0 +1,9 @@
+# blockless-car
+
+This is a Rust implementation of the CAR specifications.
+
+This project support many utils like "ls" "cat" "archive" "extract", all of them in examples.
+
+Easy to use them
+
+there are the command line tools https://github.com/blocklessnetwork/car-utils

crates/blockless-car/examples/archive.rs

@@ -0,0 +1,12 @@
+use blockless_car::utils::archive_local;
+
+/// Cat the file in car file by file id
+/// e.g. ```cargo run -p blockless-car --example `archive directory` `target car file`.```
+fn main() {
+    let file_name = std::env::args().nth(1).expect("use directory as argument");
+    let target = std::env::args()
+        .nth(2)
+        .expect("need the target file as argument");
+    let file = std::fs::File::create(target).unwrap();
+    archive_local(file_name, file).unwrap();
+}

crates/blockless-car/examples/cat_cid.rs

@@ -0,0 +1,34 @@
+use cid::Cid;
+use blockless_car::{
+    error::CarError,
+    reader::{self, CarReader},
+    unixfs::UnixFs,
+    utils::cat_ipld,
+};
+
+/// Cat the file in car file by file id
+/// e.g. ```cargo run -p blockless-car --example cat_file bafkreiabltrd5zm73pvi7plq25pef3hm7jxhbi3kv4hapegrkfpkqtkbme```
+/// the example cat file with cid in carv1-basic.car
+fn main() {
+    // let cid = std::env::args().nth(1).expect("use cid as argument");
+    let cid = "bafkreiaqv66m5nd6mwgkk7h5lwqnjzj54s4f7knmnrjhb7ylzqfg2vdo54";
+    let file = std::path::Path::new("test");
+    let file = file.join("carv1-basic.car");
+    let file = std::fs::File::open(file).unwrap();
+    let mut reader = reader::new_v1(file).unwrap();
+    let roots = reader.header().roots();
+    let file_cid = Cid::try_from(cid).expect("cid format error");
+    for r in roots.iter() {
+        let root_ipld = reader.ipld(r).unwrap();
+        let root: Result<UnixFs, CarError> = root_ipld.try_into();
+        let root_dir = root.unwrap();
+        let count = root_dir
+            .links()
+            .iter()
+            .filter(|u| u.hash() == file_cid)
+            .count();
+        if count > 0 {
+            cat_ipld(&mut reader, file_cid).unwrap();
+        }
+    }
+}

crates/blockless-car/examples/cat_file.rs

@@ -0,0 +1,19 @@
+use blockless_car::{
+    reader::{self, CarReader},
+    utils::cat_ipld,
+};
+
+/// Cat the file in car file by file id
+/// e.g. ```cargo run -p blockless-car --example cat_file file name.```
+/// the example cat used file is carv1-basic.car
+fn main() {
+    let file_name = std::env::args().nth(1).expect("use filename as argument");
+    let file = std::path::Path::new("test");
+    let file = file.join("carv1-basic.car");
+    let file = std::fs::File::open(file).unwrap();
+    let mut reader = reader::new_v1(file).unwrap();
+    let cid = reader
+        .search_file_cid(&file_name)
+        .expect("search file error.");
+    cat_ipld(&mut reader, cid).unwrap();
+}

crates/blockless-car/examples/extract.rs

@@ -0,0 +1,21 @@
+use blockless_car::{
+    reader::{self, CarReader},
+    utils::extract_ipld_to_current_path,
+};
+
+/// extract all files in car file by file id
+/// e.g. ```cargo run -p blockless-car --example extract```
+/// the example cat used file is carv1-basic.car
+fn main() {
+    let file_name = std::env::args().nth(1);
+    let path = file_name.as_ref().map(|f| f.into()).unwrap_or_else(|| {
+        let file = std::path::Path::new("test");
+        file.join("carv1-basic.car")
+    });
+    let file = std::fs::File::open(path).unwrap();
+    let mut reader = reader::new_v1(file).unwrap();
+    let roots = reader.header().roots();
+    for r in roots.iter() {
+        extract_ipld_to_current_path(&mut reader, *r).unwrap();
+    }
+}

crates/blockless-car/examples/ls.rs

@@ -0,0 +1,14 @@
+use blockless_car::reader;
+
+/// e.g. ```cargo run -p blockless-car --example ls file```
+/// the example list file infomation in carv1-basic.car file
+fn main() {
+    let file_name = std::env::args().nth(1);
+    let path = file_name.as_ref().map(|f| f.into()).unwrap_or_else(|| {
+        let file = std::path::Path::new("test");
+        file.join("carv1-basic.car")
+    });
+    let file = std::fs::File::open(path).unwrap();
+    let mut reader = reader::new_v1(file).unwrap();
+    blockless_car::utils::list(&mut reader).unwrap();
+}

crates/blockless-car/src/codec.rs

@@ -0,0 +1,9 @@
+use crate::error::CarError;
+
+pub trait Decoder<T> {
+    fn decode(&self) -> Result<T, CarError>;
+}
+
+pub trait Encoder<T> {
+    fn encode(&self) -> Result<T, CarError>;
+}

crates/blockless-car/src/error.rs

@@ -0,0 +1,22 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum CarError {
+    #[error("parsing the file error: {0}")]
+    Parsing(String),
+
+    #[error("invalid file error: {0}")]
+    InvalidFile(String),
+
+    #[error("invalid section error: {0}")]
+    InvalidSection(String),
+
+    #[error("Io error: {0}")]
+    IO(#[from] std::io::Error),
+
+    #[error("too large section error: {0}")]
+    TooLargeSection(usize),
+
+    #[error("Not found {0}")]
+    NotFound(String),
+}

crates/blockless-car/src/header.rs

@@ -0,0 +1,69 @@
+mod header_v1;
+use std::io;
+
+pub(crate) use header_v1::CarHeaderV1;
+
+use cid::Cid;
+use ipld::prelude::Codec;
+use ipld_cbor::DagCborCodec;
+
+use crate::{error::CarError, reader::read_block};
+
+#[derive(Clone, Debug)]
+pub enum CarHeader {
+    V1(CarHeaderV1),
+    V2(),
+}
+
+impl CarHeader {
+    pub fn new_v1(roots: Vec<Cid>) -> Self {
+        CarHeader::V1(CarHeaderV1::new(roots))
+    }
+
+    pub fn roots(&self) -> Vec<Cid> {
+        match *self {
+            CarHeader::V1(ref v1) => v1.roots.clone(),
+            CarHeader::V2() => todo!(),
+        }
+    }
+
+    pub fn read_header<R>(r: R) -> Result<CarHeader, CarError>
+    where
+        R: io::Read + io::Seek,
+    {
+        let data = match read_block(r) {
+            Ok(Some(d)) => d,
+            Ok(None) => return Err(CarError::Parsing("Invalid Header".into())),
+            Err(e) => return Err(e),
+        };
+        let header = CarHeader::decode(&data[..])?;
+        Ok(header)
+    }
+
+    pub fn decode(buf: &[u8]) -> Result<CarHeader, CarError> {
+        let header: CarHeaderV1 = DagCborCodec
+            .decode(buf)
+            .map_err(|e| CarError::Parsing(e.to_string()))?;
+        if header.roots.is_empty() {
+            return Err(CarError::Parsing("car roots is empty".to_owned()));
+        }
+        if header.version != 1 {
+            return Err(CarError::InvalidFile(
+                "Now CAR version 1 is supported only".to_string(),
+            ));
+        }
+        Ok(CarHeader::V1(header))
+    }
+
+    pub fn encode(&self) -> Result<Vec<u8>, CarError> {
+        match *self {
+            CarHeader::V1(ref v1) => {
+                let data = DagCborCodec
+                    .encode(v1)
+                    .map_err(|e| CarError::Parsing(e.to_string()))?;
+                Ok(data)
+            }
+            CarHeader::V2() => todo!(),
+        }
+    }
+}

crates/blockless-car/src/header/header_v1.rs

@@ -0,0 +1,43 @@
+use cid::Cid;
+
+#[derive(Debug, Clone, PartialEq, Eq, Default, ipld::DagCbor)]
+pub struct CarHeaderV1 {
+    #[ipld]
+    pub roots: Vec<Cid>,
+    #[ipld]
+    pub version: u64,
+}
+
+impl CarHeaderV1 {
+    pub fn new(roots: Vec<Cid>) -> Self {
+        Self { roots, version: 1 }
+    }
+}
+
+impl From<Vec<Cid>> for CarHeaderV1 {
+    fn from(roots: Vec<Cid>) -> Self {
+        Self::new(roots)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use cid::multihash::{Code::Blake2b256, MultihashDigest};
+    use cid::Cid;
+    use ipld::codec::{Decode, Encode};
+    use ipld_cbor::DagCborCodec;
+
+    #[test]
+    fn test_head_v1() {
+        let digest = Blake2b256.digest(b"test");
+        let cid = Cid::new_v1(DagCborCodec.into(), digest);
+        let mut bytes = Vec::new();
+        let header = CarHeaderV1::new(vec![cid]);
+        header.encode(DagCborCodec, &mut bytes).unwrap();
+        assert_eq!(
+            CarHeaderV1::decode(DagCborCodec, &mut std::io::Cursor::new(&bytes)).unwrap(),
+            header
+        );
+    }
+}

crates/blockless-car/src/lib.rs

@@ -0,0 +1,15 @@
+pub mod codec;
+pub mod error;
+pub mod header;
+mod pb;
+pub mod reader;
+pub mod section;
+pub mod unixfs;
+mod unixfs_codec;
+pub mod utils;
+pub mod writer;
+
+pub use codec::Decoder;
+pub use header::CarHeader;
+
+pub type Ipld = ipld::Ipld;

crates/blockless-car/src/pb.rs

@@ -0,0 +1 @@
+pub mod unixfs;

crates/blockless-car/src/pb/unixfs.proto

@@ -0,0 +1,34 @@
+syntax = "proto2";
+
+package unixfs.pb;
+
+message Data {
+	enum DataType {
+		Raw = 0;
+		Directory = 1;
+		File = 2;
+		Metadata = 3;
+		Symlink = 4;
+		HAMTShard = 5;
+	}
+
+	required DataType Type = 1;
+	optional bytes Data = 2;
+	optional uint64 filesize = 3;
+	repeated uint64 blocksizes = 4;
+
+	optional uint64 hashType = 5;
+	optional uint64 fanout = 6;
+
+	optional uint32 mode = 7;
+	optional UnixTime mtime = 8;
+}
+
+message UnixTime {
+	required int64 Seconds = 1;
+	optional fixed32 FractionalNanoseconds = 2;
+}
+
+message Metadata {
+	optional string MimeType = 1;
+}