Spark 3.5: Remove obsolete conf parsing logic (#10309)

apache · May 11, 2024 · 04792cf · 04792cf
1 parent e484f0d
commit 04792cf
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 14 deletions.
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
@@ -20,7 +20,6 @@
 
 import java.time.Duration;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 import org.apache.iceberg.Table;
@@ -31,23 +30,24 @@
 import org.apache.spark.sql.RuntimeConfig;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 class SparkConfParser {
 
   private final Map<String, String> properties;
   private final RuntimeConfig sessionConf;
-  private final Map<String, String> options;
+  private final CaseInsensitiveStringMap options;
 
   SparkConfParser() {
     this.properties = ImmutableMap.of();
     this.sessionConf = new RuntimeConfig(SQLConf.get());
-    this.options = ImmutableMap.of();
+    this.options = CaseInsensitiveStringMap.empty();
   }
 
   SparkConfParser(SparkSession spark, Table table, Map<String, String> options) {
     this.properties = table.properties();
     this.sessionConf = spark.conf();
-    this.options = options;
+    this.options = asCaseInsensitiveStringMap(options);
   }
 
   public BooleanConfParser booleanConf() {
@@ -70,6 +70,14 @@ public DurationConfParser durationConf() {
     return new DurationConfParser();
   }
 
+  private static CaseInsensitiveStringMap asCaseInsensitiveStringMap(Map<String, String> map) {
+    if (map instanceof CaseInsensitiveStringMap) {
+      return (CaseInsensitiveStringMap) map;
+    } else {
+      return new CaseInsensitiveStringMap(map);
+    }
+  }
+
   class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
     private Boolean defaultValue;
     private boolean negate = false;
@@ -220,14 +228,10 @@ public ThisT tableProperty(String name) {
     }
 
     protected T parse(Function<String, T> conversion, T defaultValue) {
-      if (!optionNames.isEmpty()) {
-        for (String optionName : optionNames) {
-          // use lower case comparison as DataSourceOptions.asMap() in Spark 2 returns a lower case
-          // map
-          String optionValue = options.get(optionName.toLowerCase(Locale.ROOT));
-          if (optionValue != null) {
-            return conversion.apply(optionValue);
-          }
+      for (String optionName : optionNames) {
+        String optionValue = options.get(optionName);
+        if (optionValue != null) {
+          return conversion.apply(optionValue);
         }
       }
 

diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -57,7 +57,6 @@ public class SparkReadConf {
   private final SparkSession spark;
   private final Table table;
   private final String branch;
-  private final Map<String, String> readOptions;
   private final SparkConfParser confParser;
 
   public SparkReadConf(SparkSession spark, Table table, Map<String, String> readOptions) {
@@ -69,7 +68,6 @@ public SparkReadConf(
     this.spark = spark;
     this.table = table;
     this.branch = branch;
-    this.readOptions = readOptions;
     this.confParser = new SparkConfParser(spark, table, readOptions);
   }
 

diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
@@ -79,6 +79,15 @@ public void after() {
     sql("DROP TABLE IF EXISTS %s", tableName);
   }
 
+  @TestTemplate
+  public void testOptionCaseInsensitive() {
+    Table table = validationCatalog.loadTable(tableIdent);
+    Map<String, String> options = ImmutableMap.of("option", "value");
+    SparkConfParser parser = new SparkConfParser(spark, table, options);
+    String parsedValue = parser.stringConf().option("oPtIoN").parseOptional();
+    assertThat(parsedValue).isEqualTo("value");
+  }
+
   @TestTemplate
   public void testDurationConf() {
     Table table = validationCatalog.loadTable(tableIdent);