Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Implement float*interval and interval/float #163

Merged
merged 2 commits into from
Jul 22, 2024
Merged

feat: Implement float*interval and interval/float #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fc20b3b
chore: Bump arrow to 8fd2aa80114d5c0d4e6a0c370729507a4424e7b3
mcheshkov Jul 22, 2024
4f3efdb
feat: Implement float*interval and interval/float
mcheshkov Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion datafusion-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ repository = "https://github.com/apache/arrow-datafusion"
rust-version = "1.59"

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf" }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3" }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "7.0.0" }
dirs = "4.0.0"
Expand Down
2 changes: 1 addition & 1 deletion datafusion-examples/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]

[dev-dependencies]
arrow-flight = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf" }
arrow-flight = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3" }
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
Expand Down
4 changes: 2 additions & 2 deletions datafusion/common/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ jit = ["cranelift-module"]
pyarrow = ["pyo3"]

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
cranelift-module = { version = "0.82.0", optional = true }
ordered-float = "2.10"
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["arrow"], optional = true }
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["arrow"], optional = true }
pyo3 = { version = "0.16", optional = true }
sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "6a54d27d3b75a04b9f9cbe309a83078aa54b32fd" }
4 changes: 2 additions & 2 deletions datafusion/core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions"]

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
async-trait = "0.1.41"
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
chrono = { version = "0.4", default-features = false }
Expand All @@ -73,7 +73,7 @@ num-traits = { version = "0.2", optional = true }
num_cpus = "1.13.0"
ordered-float = "2.10"
parking_lot = "0.12"
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["arrow"] }
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["arrow"] }
paste = "^1.0"
pin-project-lite= "^0.2.7"
pyo3 = { version = "0.16", optional = true }
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/fuzz-utils/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
158 changes: 158 additions & 0 deletions datafusion/core/tests/sql/expr.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,164 @@ async fn test_interval_expressions() -> Result<()> {
Ok(())
}

#[tokio::test]
async fn test_interval_mul_div_float() -> Result<()> {
macro_rules! test_mul {
($L:expr, $R:expr, $EXPECTED:expr) => {
test_expression!(format!("({}) * ({})", $L, $R), $EXPECTED);
test_expression!(format!("({}) * ({})", $R, $L), $EXPECTED);
};
}

test_mul!(
"0.5",
"interval '1 month'",
"0 years 0 mons 15 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"1.0",
"interval '1 month'",
"0 years 1 mons 0 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"1.5",
"interval '1 month'",
"0 years 1 mons 15 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"2.0",
"interval '1 month'",
"0 years 2 mons 0 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"-1.5",
"interval '1 month'",
"0 years -1 mons -15 days 0 hours 0 mins 0.000000000 secs"
);
test_expression!(
"1.5 * (interval '1 month') / 1.5",
"0 years 0 mons 30 days 0 hours 0 mins 0.000000000 secs"
);

test_mul!(
"0.5",
"interval '1 day'",
"0 years 0 mons 0 days 12 hours 0 mins 0.000000000 secs"
);
test_mul!(
"1.0",
"interval '1 day'",
"0 years 0 mons 1 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"1.5",
"interval '1 day'",
"0 years 0 mons 1 days 12 hours 0 mins 0.000000000 secs"
);
test_mul!(
"2.0",
"interval '1 day'",
"0 years 0 mons 2 days 0 hours 0 mins 0.000000000 secs"
);
test_mul!(
"-1.5",
"interval '1 day'",
"0 years 0 mons -1 days -12 hours 0 mins 0.000000000 secs"
);
test_expression!(
"1.5 * (interval '1 day') / 1.5",
"0 years 0 mons 0 days 24 hours 0 mins 0.000000000 secs"
);

test_mul!(
"0.5",
"interval '1 second'",
"0 years 0 mons 0 days 0 hours 0 mins 0.500000000 secs"
);
test_mul!(
"1.0",
"interval '1 second'",
"0 years 0 mons 0 days 0 hours 0 mins 1.000000000 secs"
);
test_mul!(
"1.5",
"interval '1 second'",
"0 years 0 mons 0 days 0 hours 0 mins 1.500000000 secs"
);
test_mul!(
"2.0",
"interval '1 second'",
"0 years 0 mons 0 days 0 hours 0 mins 2.000000000 secs"
);
// This test prints as -1.-500000000 secs, that looks like a bug in printing
// TODO fix it
// test_mul!(
// "-1.5",
// "interval '1 second'",
// "0 years 0 mons 0 days 0 hours 0 mins -1.500000000 secs"
srh marked this conversation as resolved.
Show resolved Hide resolved
// );
test_expression!(
"1.5 * (interval '1 second') / 1.5",
"0 years 0 mons 0 days 0 hours 0 mins 1.000000000 secs"
);

// Carry-over cases
test_mul!(
"1.5",
"interval '1 month 1 day'",
"0 years 1 mons 16 days 12 hours 0 mins 0.000000000 secs"
);
test_mul!(
"1.5",
"interval '1 month 1 second'",
"0 years 1 mons 15 days 0 hours 0 mins 1.500000000 secs"
);
test_mul!(
"1.5",
"interval '1 day 1 second'",
"0 years 0 mons 1 days 12 hours 0 mins 1.500000000 secs"
);

Ok(())
}

#[tokio::test]
async fn test_interval_mul_bad_float() -> Result<()> {
macro_rules! test_expr_error {
($SQL:expr, $EXPECTED:expr) => {
let ctx = SessionContext::new();
let sql = format!("SELECT {}", $SQL);
let actual_err = plan_and_collect(&ctx, sql.as_str()).await.unwrap_err();
assert_eq!(actual_err.to_string(), $EXPECTED);
};
}
macro_rules! test_mul_error {
($L:expr, $R:expr, $EXPECTED:expr) => {
test_expr_error!(format!("({}) * ({})", $L, $R), $EXPECTED);
test_expr_error!(format!("({}) * ({})", $R, $L), $EXPECTED);
};
}

// This behaviour was checked on PostgreSQL 15.7
test_mul_error!(
"cast('NaN' as double precision)",
"interval '1 month'",
"Execution error: interval out of range (float)"
);
test_mul_error!(
"cast('inf' as double precision)",
"interval '1 month'",
"Execution error: interval out of range (float)"
);
test_mul_error!(
"cast('-inf' as double precision)",
"interval '1 month'",
"Execution error: interval out of range (float)"
);

Ok(())
}

#[cfg(feature = "unicode_expressions")]
#[tokio::test]
async fn test_substring_expr() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion datafusion/cube_ext/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ name = "cube_ext"
path = "src/lib.rs"

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
chrono = { version = "0.4.16", package = "chrono", default-features = false, features = ["clock"] }
datafusion-common = { path = "../common", version = "7.0.0" }
datafusion-expr = { path = "../expr", version = "7.0.0" }
2 changes: 1 addition & 1 deletion datafusion/expr/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ path = "src/lib.rs"

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "7.0.0" }
sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "6a54d27d3b75a04b9f9cbe309a83078aa54b32fd" }
14 changes: 14 additions & 0 deletions datafusion/expr/src/binary_rule.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,20 @@ pub fn distinct_coercion(
| (Interval(unit), UInt8)
| (Null, Interval(unit))
| (Interval(unit), Null) => Some(Interval(unit.clone())),
// float*interval result is always represented as MonthDayNano, to avoid precision loss
(Float64, Interval(_))
| (Interval(_), Float64)
| (Float32, Interval(_))
| (Interval(_), Float32)
| (Float16, Interval(_))
| (Interval(_), Float16) => Some(Interval(MonthDayNano)),
_ => None,
},
Operator::Divide => match (lhs_type, rhs_type) {
// interval/float result is represented as MonthDayNano, to avoid precision loss
(Interval(_), Float64) | (Interval(_), Float32) | (Interval(_), Float16) => {
Some(Interval(MonthDayNano))
}
_ => None,
},
_ => None,
Expand Down
2 changes: 1 addition & 1 deletion datafusion/jit/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf" }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3" }
cranelift = "0.82.0"
cranelift-jit = "0.82.0"
cranelift-module = "0.82.0"
Expand Down
2 changes: 1 addition & 1 deletion datafusion/physical-expr/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "db18b5c398ba5554fc774544cec4b713445102bf", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "8fd2aa80114d5c0d4e6a0c370729507a4424e7b3", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4.20", default-features = false }
Expand Down
Loading
Loading