diff --git a/Cargo.lock b/Cargo.lock index bdf7141..2658485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "async-trait" @@ -66,7 +66,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -77,9 +77,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -115,7 +115,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.64", + "syn 2.0.66", "which", ] @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" [[package]] name = "cexpr" @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -310,9 +310,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "env_logger" @@ -402,7 +402,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -486,9 +486,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -504,8 +504,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hdfs-native" -version = "0.9.1" -source = "git+https://github.com/Kimahriman/hdfs-native.git#20f23c3827e9d454e96e1517017af99703c7cab9" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d177f038e3d75aa76d5be274af7a80e519a2974ac8d927cf71871849aa0f018" dependencies = [ "aes", "base64", @@ -542,7 +543,7 @@ dependencies = [ [[package]] name = "hdfs-native-object-store" -version = "0.9.1" +version = "0.10.0" dependencies = [ "async-trait", "bytes", @@ -684,14 +685,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libgssapi" -version = "0.7.0" -source = "git+https://github.com/Kimahriman/libgssapi.git?branch=raw-slice-panic#7ed51d50a004fa2c8ea60041c44c6abdde122016" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c502363cc9845707a9b9fa938b732111fa7d7a4e518ebda8e46c5a1471192a15" dependencies = [ "bitflags", "bytes", @@ -702,7 +704,8 @@ dependencies = [ [[package]] name = "libgssapi-sys" version = "0.3.1" -source = "git+https://github.com/Kimahriman/libgssapi.git?branch=raw-slice-panic#7ed51d50a004fa2c8ea60041c44c6abdde122016" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57d9a71c774ec53b1b9119dbbcc589b5209831f0ddc0ff4210640051f545372" dependencies = [ "bindgen", "pkg-config", @@ -720,9 +723,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" @@ -764,9 +767,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -813,9 +816,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -849,9 +852,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -907,23 +910,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -931,22 +934,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -1099,7 +1102,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1174,9 +1177,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.64" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad3dee41f36859875573074334c200d1add8e4a87bb37113ebd31d926b7b11f" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -1194,22 +1197,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1229,9 +1232,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -1246,13 +1249,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1274,7 +1277,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1385,7 +1388,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -1407,7 +1410,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 48980d3..23f390c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hdfs-native-object-store" -version = "0.9.1" +version = "0.10.0" edition = "2021" authors = ["Adam Binford "] homepage = "https://github.com/datafusion-contrib/hdfs-native-object-store" @@ -15,8 +15,7 @@ async-trait = "0.1" bytes = "1" chrono = "0.4" futures = "0.3" -# Need master branch until next version is released for tests to pass -hdfs-native = { version = "0.9", git = "https://github.com/Kimahriman/hdfs-native.git" } +hdfs-native = "0.9" object_store = "0.10" thiserror = "1" tokio = { version = "1", features = ["rt", "net", "io-util", "macros", "sync", "time"] } diff --git a/README.md b/README.md index d848130..9d42481 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,22 @@ -# Object store implementation for the Native Rust HDFS client +# HDFS Native Object Store +An [object_store](https://docs.rs/object_store/latest/object_store/) implementation for HDFS based on the native Rust [hdfs-native](https://github.com/Kimahriman/hdfs-native) library. -## Usage +# Compatibility +Each release supports a certain minor release of both the `object_store` crate and the underlying `hdfs-native` client. +|hdfs-native-object-store|object_store|hdfs-native| +|---|---|---| +|0.9.x|0.9|0.9| +|0.10.x|0.10|0.9| + +# Usage ```rust use hdfs_native_object_store::HdfsObjectStore; -use hdfs_native::Result; -fn main() -> Result<()> { - let store = HdfsObjectStore::with_url("hdfs://localhost:9000")?; - Ok(()) -} -``` \ No newline at end of file +let store = HdfsObjectStore::with_url("hdfs://localhost:9000")?; +``` + +# Feature Flags +`kerberos` - Enables Kerberos authentication support via the [libgssapi](https://docs.rs/libgssapi/latest/libgssapi) crate + +# Documentation +See [Documentation](https://docs.rs/hdfs-native-object-store). \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index f4a280f..73e491e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ // #![warn(missing_docs)] -//! Object store implementation for the Native Rust HDFS client +//! [object_store::ObjectStore] implementation for the Native Rust HDFS client //! //! # Usage //! @@ -11,6 +11,10 @@ //! # Ok(()) //! # } //! ``` +//! +//! # Feature Flags +//! `kerberos` - Enables Kerberos authentication support via the [libgssapi](https://docs.rs/libgssapi/latest/libgssapi) crate +//! use std::{ fmt::{Display, Formatter}, future, @@ -41,10 +45,38 @@ pub struct HdfsObjectStore { } impl HdfsObjectStore { + /// Creates a new HdfsObjectStore from an existing [Client] + /// + /// ```rust + /// # use std::sync::Arc; + /// use hdfs_native::Client; + /// # use hdfs_native_object_store::HdfsObjectStore; + /// let client = Client::new("hdfs://127.0.0.1:9000").unwrap(); + /// let store = HdfsObjectStore::new(Arc::new(client)); + /// ``` pub fn new(client: Arc) -> Self { Self { client } } + /// Creates a new HdfsObjectStore using the specified URL + /// + /// Connect to a single NameNode + /// ```rust + /// # use hdfs_native_object_store::HdfsObjectStore; + /// # fn main() -> object_store::Result<()> { + /// let store = HdfsObjectStore::with_url("hdfs://127.0.0.1:9000")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// Connect to a NameService + /// ```rust + /// # use hdfs_native_object_store::HdfsObjectStore; + /// # fn main() -> object_store::Result<()> { + /// let store = HdfsObjectStore::with_url("hdfs://ns")?; + /// # Ok(()) + /// # } + /// ``` pub fn with_url(url: &str) -> Result { Ok(Self { client: Arc::new(Client::new(url).to_object_store_err()?), @@ -133,8 +165,9 @@ impl From for HdfsObjectStore { impl ObjectStore for HdfsObjectStore { /// Save the provided bytes to the specified location /// - /// To make the operation atomic, we write to a temporary file ".{filename}.tmp.{i}" and rename - /// on a successful write. + /// To make the operation atomic, we write to a temporary file `.{filename}.tmp.{i}` and rename + /// on a successful write, where `i` is an integer that is incremented until a non-existent file + /// is found. async fn put_opts( &self, location: &Path, @@ -197,6 +230,7 @@ impl ObjectStore for HdfsObjectStore { ))) } + /// Reads data for the specified location. async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { if options.if_match.is_some() || options.if_none_match.is_some() @@ -345,6 +379,7 @@ impl ObjectStore for HdfsObjectStore { }) } + /// Renames a file. This operation is guaranteed to be atomic. async fn rename(&self, from: &Path, to: &Path) -> Result<()> { Ok(self .client @@ -353,6 +388,8 @@ impl ObjectStore for HdfsObjectStore { .to_object_store_err()?) } + /// Renames a file only if the distination doesn't exist. This operation is guaranteed + /// to be atomic. async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> { Ok(self .client