feat: Implement Lazy Reader (#3395)

Signed-off-by: Xuanwo <[email protected]>
apache · Oct 31, 2023 · 268d5b8 · 268d5b8
1 parent 74eab84
commit 268d5b8
Show file tree

Hide file tree

Showing 79 changed files with 2,537 additions and 1,635 deletions.
diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml
@@ -42,7 +42,7 @@ runs:
         # Enable backtraces
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         # Enable logging
-        echo "RUST_LOG=opendal=debug" >> $GITHUB_ENV
+        echo "RUST_LOG=opendal=trace" >> $GITHUB_ENV
         # Enable sparse index
         echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> $GITHUB_ENV
 

diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml
@@ -102,6 +102,7 @@ jobs:
     with:
       os: ${{ matrix.os }}
       cases: ${{ toJson(matrix.cases) }}
+
   test_binding_python:
     name: binding_python / ${{ matrix.os }}
     needs: [plan]

diff --git a/.typos.toml b/.typos.toml
@@ -18,8 +18,8 @@
 [default.extend-words]
 # Random strings.
 "Dum" = "Dum"
-"ba" = "ba"
 "Hel" = "Hel"
+"ba" = "ba"
 "hellow" = "hellow"
 # Showed up in examples.
 "thw" = "thw"
diff --git a/bin/oay/Cargo.toml b/bin/oay/Cargo.toml
@@ -33,7 +33,12 @@ version.workspace = true
 default = ["frontends-webdav", "frontends-s3"]
 
 frontends-s3 = []
-frontends-webdav = ["dep:dav-server", "dep:dav-server-opendalfs", "dep:bytes", "dep:futures-util"]
+frontends-webdav = [
+  "dep:dav-server",
+  "dep:dav-server-opendalfs",
+  "dep:bytes",
+  "dep:futures-util",
+]
 
 [dependencies]
 anyhow = "1"

diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml
@@ -36,6 +36,6 @@ cbindgen = "0.25.0"
 
 [dependencies]
 bytes = "1.4.0"
+once_cell = "1.17.1"
 opendal.workspace = true
 tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] }
-once_cell = "1.17.1"
diff --git a/bindings/cpp/Cargo.toml b/bindings/cpp/Cargo.toml
@@ -24,17 +24,17 @@ edition.workspace = true
 homepage.workspace = true
 license.workspace = true
 repository.workspace = true
-version.workspace = true
 rust-version.workspace = true
+version.workspace = true
 
 [lib]
 crate-type = ["staticlib"]
 
 [dependencies]
-opendal.workspace = true
-cxx = "1.0"
 anyhow = "1.0"
 chrono = "0.4"
+cxx = "1.0"
+opendal.workspace = true
 
 [build-dependencies]
 cxx-build = "1.0"
diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml
@@ -27,7 +27,6 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
-
 [lib]
 crate-type = ["cdylib"]
 doc = false

diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml
@@ -24,8 +24,8 @@ edition.workspace = true
 homepage.workspace = true
 license.workspace = true
 repository.workspace = true
-version.workspace = true
 rust-version.workspace = true
+version.workspace = true
 
 [lib]
 crate-type = ["cdylib"]

diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml
@@ -86,22 +86,23 @@ services-all = [
 ]
 
 # Default services provided by opendal.
-services-azblob = [ "opendal/services-azblob" ]
-services-azdls = [ "opendal/services-azdls" ]
-services-cos = [ "opendal/services-cos" ]
-services-fs = [ "opendal/services-fs" ]
-services-gcs = [ "opendal/services-gcs" ]
-services-ghac = [ "opendal/services-ghac" ]
-services-http = [ "opendal/services-http" ]
-services-ipmfs = [ "opendal/services-ipmfs" ]
-services-memory = [ "opendal/services-memory" ]
-services-obs = [ "opendal/services-obs" ]
-services-oss = [ "opendal/services-oss" ]
-services-s3 = [ "opendal/services-s3" ]
-services-webdav = [ "opendal/services-webdav" ]
-services-webhdfs = [ "opendal/services-webhdfs" ]
+services-azblob = ["opendal/services-azblob"]
+services-azdls = ["opendal/services-azdls"]
+services-cos = ["opendal/services-cos"]
+services-fs = ["opendal/services-fs"]
+services-gcs = ["opendal/services-gcs"]
+services-ghac = ["opendal/services-ghac"]
+services-http = ["opendal/services-http"]
+services-ipmfs = ["opendal/services-ipmfs"]
+services-memory = ["opendal/services-memory"]
+services-obs = ["opendal/services-obs"]
+services-oss = ["opendal/services-oss"]
+services-s3 = ["opendal/services-s3"]
+services-webdav = ["opendal/services-webdav"]
+services-webhdfs = ["opendal/services-webhdfs"]
 
 # Optional services provided by opendal.
+services-azfile = ["opendal/services-azfile"]
 services-cacache = ["opendal/services-cacache"]
 services-dashmap = ["opendal/services-dashmap"]
 services-dropbox = ["opendal/services-dropbox"]
@@ -114,6 +115,8 @@ services-ipfs = ["opendal/services-ipfs"]
 services-memcached = ["opendal/services-memcached"]
 services-mini-moka = ["opendal/services-mini-moka"]
 services-moka = ["opendal/services-moka"]
+services-mongodb = ["opendal/services-mongodb"]
+services-mysql = ["opendal/services-mysql"]
 services-onedrive = ["opendal/services-onedrive"]
 services-persy = ["opendal/services-persy"]
 services-postgresql = ["opendal/services-postgresql"]
@@ -123,28 +126,25 @@ services-redis-rustls = ["opendal/services-redis-rustls"]
 services-rocksdb = ["opendal/services-rocksdb"]
 services-sftp = ["opendal/services-sftp"]
 services-sled = ["opendal/services-sled"]
+services-sqlite = ["opendal/services-sqlite"]
 services-supabase = ["opendal/services-supabase"]
 services-tikv = ["opendal/services-tikv"]
 services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
 services-wasabi = ["opendal/services-wasabi"]
-services-mysql = ["opendal/services-mysql"]
-services-mongodb = ["opendal/services-mongodb"]
-services-sqlite = ["opendal/services-sqlite"]
-services-azfile = ["opendal/services-azfile"]
 
 [dependencies]
 anyhow = "1.0.71"
 jni = "0.21.1"
 num_cpus = "1.15.0"
 once_cell = "1.17.1"
-tokio = { version = "1.28.1", features = ["full"] }
 opendal = { workspace = true }
+tokio = { version = "1.28.1", features = ["full"] }
 
 # This is not optimal. See also the Cargo issue:
 # https://github.com/rust-lang/cargo/issues/1197#issuecomment-1641086954
 [target.'cfg(unix)'.dependencies.opendal]
-workspace = true
 features = [
   # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows.
   "services-sftp",
 ]
+workspace = true
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -27,7 +27,6 @@ repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
 
-
 [features]
 # Enable all opendal default feature by default.
 default = [
@@ -50,6 +49,7 @@ default = [
 services-all = [
   "default",
 
+  "services-azfile",
   "services-cacache",
   "services-dashmap",
   "services-dropbox",
@@ -82,20 +82,21 @@ services-all = [
 ]
 
 # Default services provided by opendal.
-services-azblob = [ "opendal/services-azblob" ]
-services-azdls = [ "opendal/services-azdls" ]
-services-cos = [ "opendal/services-cos" ]
-services-fs = [ "opendal/services-fs" ]
-services-gcs = [ "opendal/services-gcs" ]
-services-ghac = [ "opendal/services-ghac" ]
-services-http = [ "opendal/services-http" ]
-services-ipmfs = [ "opendal/services-ipmfs" ]
-services-memory = [ "opendal/services-memory" ]
-services-obs = [ "opendal/services-obs" ]
-services-oss = [ "opendal/services-oss" ]
-services-s3 = [ "opendal/services-s3" ]
-services-webdav = [ "opendal/services-webdav" ]
-services-webhdfs = [ "opendal/services-webhdfs" ]
+services-azblob = ["opendal/services-azblob"]
+services-azdls = ["opendal/services-azdls"]
+services-azfile = ["opendal/services-azfile"]
+services-cos = ["opendal/services-cos"]
+services-fs = ["opendal/services-fs"]
+services-gcs = ["opendal/services-gcs"]
+services-ghac = ["opendal/services-ghac"]
+services-http = ["opendal/services-http"]
+services-ipmfs = ["opendal/services-ipmfs"]
+services-memory = ["opendal/services-memory"]
+services-obs = ["opendal/services-obs"]
+services-oss = ["opendal/services-oss"]
+services-s3 = ["opendal/services-s3"]
+services-webdav = ["opendal/services-webdav"]
+services-webhdfs = ["opendal/services-webhdfs"]
 
 # Optional services provided by opendal.
 services-cacache = ["opendal/services-cacache"]
@@ -110,6 +111,8 @@ services-ipfs = ["opendal/services-ipfs"]
 services-memcached = ["opendal/services-memcached"]
 services-mini-moka = ["opendal/services-mini-moka"]
 services-moka = ["opendal/services-moka"]
+services-mongodb = ["opendal/services-mongodb"]
+services-mysql = ["opendal/services-mysql"]
 services-onedrive = ["opendal/services-onedrive"]
 services-persy = ["opendal/services-persy"]
 services-postgresql = ["opendal/services-postgresql"]
@@ -119,13 +122,11 @@ services-redis-rustls = ["opendal/services-redis-rustls"]
 services-rocksdb = ["opendal/services-rocksdb"]
 services-sftp = ["opendal/services-sftp"]
 services-sled = ["opendal/services-sled"]
+services-sqlite = ["opendal/services-sqlite"]
 services-supabase = ["opendal/services-supabase"]
 services-tikv = ["opendal/services-tikv"]
 services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
 services-wasabi = ["opendal/services-wasabi"]
-services-mysql = ["opendal/services-mysql"]
-services-mongodb = ["opendal/services-mongodb"]
-services-sqlite = ["opendal/services-sqlite"]
 
 [lib]
 crate-type = ["cdylib"]
@@ -136,4 +137,4 @@ futures = "0.3.28"
 opendal.workspace = true
 pyo3 = "0.19"
 pyo3-asyncio = { version = "0.19", features = ["tokio-runtime"] }
-tokio = "1"
+tokio = "1"
diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -234,6 +234,7 @@ chrono = { version = "0.4.28", default-features = false, features = [
 ] }
 dashmap = { version = "5.4", optional = true }
 dirs = { version = "5.0.1", optional = true }
+dotenvy = { version = "0.15", optional = true }
 etcd-client = { version = "0.11", optional = true, features = ["tls"] }
 flagset = "0.4"
 foundationdb = { version = "0.8.0", features = [
@@ -295,7 +296,6 @@ tokio = "1.27"
 tokio-postgres = { version = "0.7.8", optional = true }
 tracing = { version = "0.1", optional = true }
 uuid = { version = "1", features = ["serde", "v4"] }
-dotenvy = {  version = "0.15", optional = true }
 
 [dev-dependencies]
 criterion = { version = "0.4", features = ["async", "async_tokio"] }

diff --git a/core/edge/file_write_on_full_disk/Cargo.toml b/core/edge/file_write_on_full_disk/Cargo.toml
@@ -16,13 +16,13 @@
 # under the License.
 
 [package]
-name = "edge_test_file_write_on_full_disk"
 edition = "2021"
-version = "0.0.0"
+name = "edge_test_file_write_on_full_disk"
 publish = false
+version = "0.0.0"
 
 [dependencies]
 futures = "0.3"
 opendal = { workspace = true }
-tokio = { version = "1", features = ["full"] }
 rand = "0.8"
+tokio = { version = "1", features = ["full"] }
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
@@ -32,11 +32,11 @@ dotenvy = "0.15.6"
 libfuzzer-sys = "0.4"
 opendal = { path = "..", features = ["tests"] }
 tokio = { version = "1", features = ["full"] }
-uuid = { version = "1", features = ["v4"] }
 tracing-subscriber = { version = "0.3", features = [
   "env-filter",
   "tracing-log",
 ] }
+uuid = { version = "1", features = ["v4"] }
 
 [[bin]]
 name = "fuzz_reader"