Skip to content

Commit

Permalink
Round floats but not decimals in SqlLogicTests
Browse files Browse the repository at this point in the history 
- stop rounding decimal values in SLT. It's the very nature of decimal
  arithmetics that it should in general be exact.
- round float values taking into account the bit width of the float. The
  Float64 carries more information than Float32 or Float16.
  • Loading branch information
@findepi
findepi committed Dec 12, 2024
1 parent af5b84b commit 49466f2
Show file tree
Hide file tree
Showing 29 changed files with 1,307 additions and 1,244 deletions.
217 changes: 140 additions & 77 deletions datafusion/sqllogictest/src/engines/conversion.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// under the License.

use arrow::datatypes::{i256, Decimal128Type, Decimal256Type, DecimalType};
use bigdecimal::BigDecimal;
use bigdecimal::{BigDecimal, RoundingMode};
use half::f16;
use rust_decimal::prelude::*;

Expand All @@ -40,34 +40,18 @@ pub(crate) fn varchar_to_str(value: &str) -> String {
}

pub(crate) fn f16_to_str(value: f16) -> String {
if value.is_nan() {
// The sign of NaN can be different depending on platform.
// So the string representation of NaN ignores the sign.
"NaN".to_string()
} else if value == f16::INFINITY {
"Infinity".to_string()
} else if value == f16::NEG_INFINITY {
"-Infinity".to_string()
} else {
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
}
float_to_str(value.into(), 4)
}

pub(crate) fn f32_to_str(value: f32) -> String {
if value.is_nan() {
// The sign of NaN can be different depending on platform.
// So the string representation of NaN ignores the sign.
"NaN".to_string()
} else if value == f32::INFINITY {
"Infinity".to_string()
} else if value == f32::NEG_INFINITY {
"-Infinity".to_string()
} else {
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
}
float_to_str(value.into(), 6)
}

pub(crate) fn f64_to_str(value: f64) -> String {
float_to_str(value, 10)
}

fn float_to_str(value: f64, max_significant_digits: u8) -> String {
if value.is_nan() {
// The sign of NaN can be different depending on platform.
// So the string representation of NaN ignores the sign.
Expand All @@ -77,79 +61,158 @@ pub(crate) fn f64_to_str(value: f64) -> String {
} else if value == f64::NEG_INFINITY {
"-Infinity".to_string()
} else {
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
let mut big_decimal = BigDecimal::from_f64(value)
.unwrap()
// Truncate trailing decimal zeros
.normalized();
let precision = big_decimal.digits();
if precision > max_significant_digits as u64 {
let scale = big_decimal.as_bigint_and_exponent().1;
big_decimal = big_decimal
.with_scale_round(
scale + (max_significant_digits as i64 - precision as i64),
RoundingMode::HalfUp,
)
// Truncate trailing decimal zeros
.normalized();
}
big_decimal.to_plain_string()
}
}

pub(crate) fn decimal_128_to_str(value: i128, scale: i8) -> String {
let precision = u8::MAX; // does not matter
big_decimal_to_str(
BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
.unwrap(),
)
BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
.unwrap()
.to_plain_string()
}

pub(crate) fn decimal_256_to_str(value: i256, scale: i8) -> String {
let precision = u8::MAX; // does not matter
big_decimal_to_str(
BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
.unwrap(),
)
BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
.unwrap()
.to_plain_string()
}

#[cfg(feature = "postgres")]
pub(crate) fn decimal_to_str(value: Decimal) -> String {
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
}

pub(crate) fn big_decimal_to_str(value: BigDecimal) -> String {
// Round the value to limit the number of decimal places
let value = value.round(12).normalized();
// Format the value to a string
value.to_plain_string()
BigDecimal::from_str(&value.to_string())
.unwrap()
.to_plain_string()
}

#[cfg(test)]
mod tests {
use super::big_decimal_to_str;
use bigdecimal::{num_bigint::BigInt, BigDecimal};

macro_rules! assert_decimal_str_eq {
($integer:expr, $scale:expr, $expected:expr) => {
assert_eq!(
big_decimal_to_str(BigDecimal::from_bigint(
BigInt::from($integer),
$scale
)),
$expected
);
};
use super::*;
use half::f16;

#[test]
fn test_f16_to_str() {
assert_eq!(f16_to_str(f16::from_f32(0.)), "0");
assert_eq!(f16_to_str(f16::from_f32(1.)), "1");
assert_eq!(f16_to_str(f16::from_f32(12.345)), "12.34");
assert_eq!(f16_to_str(f16::from_f32(-12.345)), "-12.34");
assert_eq!(f16_to_str(f16::MAX), "65500");
assert_eq!(f16_to_str(f16::MIN), "-65500");
assert_eq!(f16_to_str(f16::EPSILON), "0.0009766");
assert_eq!(f16_to_str(f16::from_f32(f32::INFINITY)), "Infinity");
assert_eq!(f16_to_str(f16::from_f32(f32::NEG_INFINITY)), "-Infinity");
assert_eq!(f16_to_str(f16::from_f32(f32::NAN)), "NaN");
}

#[test]
fn test_f32_to_str() {
assert_eq!(f32_to_str(0.), "0");
assert_eq!(f32_to_str(1.), "1");
assert_eq!(f32_to_str(12.345), "12.345");
assert_eq!(f32_to_str(-12.345), "-12.345");
assert_eq!(f32_to_str(0.0000012345), "0.0000012345");
assert_eq!(f32_to_str(-0.0000012345), "-0.0000012345");
assert_eq!(f32_to_str(12.345678), "12.3457");
assert_eq!(f32_to_str(-12.345678), "-12.3457");
assert_eq!(
f32_to_str(f32::MAX),
"340282000000000000000000000000000000000"
);
assert_eq!(
f32_to_str(f32::MIN),
"-340282000000000000000000000000000000000"
);
assert_eq!(f32_to_str(f32::EPSILON), "0.000000119209");
assert_eq!(f32_to_str(f32::INFINITY), "Infinity");
assert_eq!(f32_to_str(f32::NEG_INFINITY), "-Infinity");
assert_eq!(f32_to_str(f32::NAN), "NaN");
}

#[test]
fn test_f64_to_str() {
assert_eq!(f64_to_str(0.), "0");
assert_eq!(f64_to_str(1.), "1");
assert_eq!(f64_to_str(12.345), "12.345");
assert_eq!(f64_to_str(-12.345), "-12.345");
assert_eq!(f64_to_str(12.345678), "12.345678");
assert_eq!(f64_to_str(-12.345678), "-12.345678");
assert_eq!(f64_to_str(0.00000000012345678), "0.00000000012345678");
assert_eq!(f64_to_str(-0.00000000012345678), "-0.00000000012345678");
assert_eq!(f64_to_str(12.34567890123456), "12.3456789");
assert_eq!(f64_to_str(-12.34567890123456), "-12.3456789");
assert_eq!(f64_to_str(0.99999999999999999999999), "1");
assert_eq!(f64_to_str(0.0000000000999999999999999), "0.0000000001");
assert_eq!(f64_to_str(f64::MAX), "179769313500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
assert_eq!(f64_to_str(f64::MIN), "-179769313500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
assert_eq!(f64_to_str(f64::EPSILON), "0.0000000000000002220446049");
assert_eq!(f64_to_str(f64::INFINITY), "Infinity");
assert_eq!(f64_to_str(f64::NEG_INFINITY), "-Infinity");
assert_eq!(f64_to_str(f64::NAN), "NaN");
}

#[test]
fn test_decimal_128_to_str() {
assert_eq!(decimal_128_to_str(1, 0), "1");
assert_eq!(decimal_128_to_str(1, 5), "0.00001");
assert_eq!(decimal_128_to_str(1, 20), "0.00000000000000000001");
assert_eq!(
decimal_128_to_str(1, 38),
"0.00000000000000000000000000000000000001"
);
assert_eq!(
decimal_128_to_str(12345678901234567890123456789012345678, 20),
"123456789012345678.90123456789012345678"
);
assert_eq!(
decimal_128_to_str(12345678901234567890123456789012345678, 0),
"12345678901234567890123456789012345678"
);
}

#[test]
fn test_big_decimal_to_str() {
assert_decimal_str_eq!(110, 3, "0.11");
assert_decimal_str_eq!(11, 3, "0.011");
assert_decimal_str_eq!(11, 2, "0.11");
assert_decimal_str_eq!(11, 1, "1.1");
assert_decimal_str_eq!(11, 0, "11");
assert_decimal_str_eq!(11, -1, "110");
assert_decimal_str_eq!(0, 0, "0");
assert_decimal_str_eq!(12345678901234567890123456789012345678_i128, 0, "12345678901234567890123456789012345678");
assert_decimal_str_eq!(12345678901234567890123456789012345678_i128, 38, "0.123456789012");

// Negative cases
assert_decimal_str_eq!(-110, 3, "-0.11");
assert_decimal_str_eq!(-11, 3, "-0.011");
assert_decimal_str_eq!(-11, 2, "-0.11");
assert_decimal_str_eq!(-11, 1, "-1.1");
assert_decimal_str_eq!(-11, 0, "-11");
assert_decimal_str_eq!(-11, -1, "-110");
assert_decimal_str_eq!(-12345678901234567890123456789012345678_i128, 0, "-12345678901234567890123456789012345678");
assert_decimal_str_eq!(-12345678901234567890123456789012345678_i128, 38, "-0.123456789012");

// Round to 12 decimal places
// 1.0000000000011 -> 1.000000000001
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, "1.000000000001");
fn test_decimal_256_to_str() {
assert_eq!(decimal_256_to_str(i256::from_str("1").unwrap(), 0), "1");
assert_eq!(
decimal_256_to_str(i256::from_str("1").unwrap(), 5),
"0.00001"
);
assert_eq!(
decimal_256_to_str(i256::from_str("1").unwrap(), 20),
"0.00000000000000000001"
);
assert_eq!(
decimal_256_to_str(i256::from_str("1").unwrap(), 38),
"0.00000000000000000000000000000000000001"
);
assert_eq!(
decimal_256_to_str(
i256::from_str("12345678901234567890123456789012345678").unwrap(),
20
),
"123456789012345678.90123456789012345678"
);
assert_eq!(
decimal_256_to_str(
i256::from_str("12345678901234567890123456789012345678").unwrap(),
0
),
"12345678901234567890123456789012345678"
);
}
}
Loading

0 comments on commit 49466f2

Please sign in to comment.