From b420d15bd5f9d884d3128968b8d96895ab972edb Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Tue, 18 Jun 2024 20:08:42 -0400 Subject: [PATCH 1/2] Upgrade hdfs-native to 0.10 --- Cargo.lock | 183 ++++++++++++++------------------------------------- Cargo.toml | 6 +- README.md | 4 +- src/error.rs | 55 ---------------- src/lib.rs | 3 - 5 files changed, 51 insertions(+), 200 deletions(-) delete mode 100644 src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 2658485..00f1cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,27 +97,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] -name = "bindgen" -version = "0.69.4" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.66", - "which", -] +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" @@ -170,15 +153,6 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -209,17 +183,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -490,12 +453,6 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "hashbrown" version = "0.14.5" @@ -504,12 +461,13 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hdfs-native" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d177f038e3d75aa76d5be274af7a80e519a2974ac8d927cf71871849aa0f018" +checksum = "7ea02e367508e4e096e07d73216ce03ec817db26dea67781c08fa855c4dd76a3" dependencies = [ "aes", "base64", + "bitflags 2.5.0", "bytes", "cbc", "chrono", @@ -522,7 +480,7 @@ dependencies = [ "hex", "hmac", "libc", - "libgssapi", + "libloading", "log", "md-5", "num-traits", @@ -536,14 +494,14 @@ dependencies = [ "thiserror", "tokio", "url", - "users", "uuid", "which", + "whoami", ] [[package]] name = "hdfs-native-object-store" -version = "0.10.0" +version = "0.11.0" dependencies = [ "async-trait", "bytes", @@ -677,40 +635,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" -[[package]] -name = "libgssapi" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c502363cc9845707a9b9fa938b732111fa7d7a4e518ebda8e46c5a1471192a15" -dependencies = [ - "bitflags", - "bytes", - "lazy_static", - "libgssapi-sys", -] - -[[package]] -name = "libgssapi-sys" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57d9a71c774ec53b1b9119dbbcc589b5209831f0ddc0ff4210640051f545372" -dependencies = [ - "bindgen", - "pkg-config", -] - [[package]] name = "libloading" version = "0.8.3" @@ -759,12 +689,6 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.7.3" @@ -785,16 +709,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -868,7 +782,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", "windows-targets 0.52.5", ] @@ -891,28 +805,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettyplease" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" -dependencies = [ - "proc-macro2", - "syn 2.0.66", -] - [[package]] name = "proc-macro2" version = "1.0.85" @@ -993,13 +891,22 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags", + "bitflags 2.5.0", ] [[package]] @@ -1046,19 +953,13 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustix" version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -1105,12 +1006,6 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "slab" version = "0.4.9" @@ -1327,15 +1222,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "users" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" -dependencies = [ - "libc", -] - [[package]] name = "uuid" version = "1.8.0" @@ -1367,6 +1253,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -1421,6 +1313,16 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" @@ -1433,6 +1335,17 @@ dependencies = [ "rustix", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + [[package]] name = "winapi-util" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 23f390c..714fa52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hdfs-native-object-store" -version = "0.10.0" +version = "0.11.0" edition = "2021" authors = ["Adam Binford "] homepage = "https://github.com/datafusion-contrib/hdfs-native-object-store" @@ -15,7 +15,7 @@ async-trait = "0.1" bytes = "1" chrono = "0.4" futures = "0.3" -hdfs-native = "0.9" +hdfs-native = "0.10" object_store = "0.10" thiserror = "1" tokio = { version = "1", features = ["rt", "net", "io-util", "macros", "sync", "time"] } @@ -25,6 +25,4 @@ env_logger = "0.10" serial_test = "2" [features] -kerberos = ["hdfs-native/kerberos"] - integration-test = ["hdfs-native/integration-test"] \ No newline at end of file diff --git a/README.md b/README.md index 9d42481..1defc07 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Each release supports a certain minor release of both the `object_store` crate a |---|---|---| |0.9.x|0.9|0.9| |0.10.x|0.10|0.9| +|0.11.x|0.10|0.10| # Usage ```rust @@ -15,8 +16,5 @@ use hdfs_native_object_store::HdfsObjectStore; let store = HdfsObjectStore::with_url("hdfs://localhost:9000")?; ``` -# Feature Flags -`kerberos` - Enables Kerberos authentication support via the [libgssapi](https://docs.rs/libgssapi/latest/libgssapi) crate - # Documentation See [Documentation](https://docs.rs/hdfs-native-object-store). \ No newline at end of file diff --git a/src/error.rs b/src/error.rs deleted file mode 100644 index e2aff56..0000000 --- a/src/error.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::io; - -#[cfg(feature = "kerberos")] -use libgssapi::error::Error as GssapiError; -use prost::DecodeError; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum HdfsError { - #[error("IO error occurred while communicating with HDFS")] - IOError(#[from] io::Error), - #[error("data transfer error")] - DataTransferError(String), - #[error("checksums didn't match")] - ChecksumError, - #[error("invalid path")] - InvalidPath(String), - #[error("invalid argument")] - InvalidArgument(String), - #[error("failed to parse URL")] - UrlParseError(#[from] url::ParseError), - #[error("file already exists")] - AlreadyExists(String), - #[error("operation failed")] - OperationFailed(String), - #[error("file not found")] - FileNotFound(String), - #[error("blocks not found")] - BlocksNotFound(String), - #[error("path is a directory")] - IsADirectoryError(String), - #[error("unsupported erasure coding policy")] - UnsupportedErasureCodingPolicy(String), - #[error("erasure coding error")] - ErasureCodingError(String), - #[error("operation not supported")] - UnsupportedFeature(String), - #[error("interal error, this shouldn't happen")] - InternalError(String), - #[error("failed to decode RPC response")] - InvalidRPCResponse(#[from] DecodeError), - #[error("RPC error")] - RPCError(String, String), - #[error("fatal RPC error")] - FatalRPCError(String, String), - #[error("SASL error")] - SASLError(String), - #[cfg(feature = "kerberos")] - #[error("GSSAPI error")] - GSSAPIError(#[from] GssapiError), - #[error("No valid SASL mechanism found")] - NoSASLMechanism, -} - -pub type Result = std::result::Result; diff --git a/src/lib.rs b/src/lib.rs index b53b5ab..341f88f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,9 +12,6 @@ //! # } //! ``` //! -//! # Feature Flags -//! `kerberos` - Enables Kerberos authentication support via the [libgssapi](https://docs.rs/libgssapi/latest/libgssapi) crate -//! use std::{ collections::HashMap, fmt::{Display, Formatter}, From 4538836a4c36259d0a15a842e20990c81ce8c225 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Tue, 18 Jun 2024 20:29:32 -0400 Subject: [PATCH 2/2] Fix CI --- .github/workflows/rust-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 2d24aed..36b9c8e 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -45,13 +45,13 @@ jobs: run: sudo apt-get install -y libkrb5-dev - name: build and lint with clippy - run: cargo clippy --all-targets --features kerberos,integration-test -- -D warnings + run: cargo clippy --all-targets --features integration-test -- -D warnings - name: Check docs run: cargo doc - name: Check all features - run: cargo check --all-targets --features kerberos,integration-test + run: cargo check --all-targets --features integration-test test: strategy: @@ -94,4 +94,4 @@ jobs: echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH - name: Run tests - run: cargo test --features kerberos,integration-test + run: cargo test --features integration-test