[GOBBLIN-1871]Fix bug that hiveMetadataWriter may make the hive schem…

…a columns inconsistent with the Avro.schema.literal
apache · Aug 8, 2023 · 3783ecb · 3783ecb
1 parent 2262d4a
commit 3783ecb
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 7 deletions.
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/orc/HiveOrcSerDeManager.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/orc/HiveOrcSerDeManager.java
@@ -255,7 +255,6 @@ private static boolean isORC(Path file, FileSystem fs)
 
   private void addSchemaProperties(Path path, HiveRegistrationUnit hiveUnit)
       throws IOException {
-    Preconditions.checkArgument(this.fs.getFileStatus(path).isDirectory(), path + " is not a directory.");
     try (Timer.Context context = metricContext.timer(HIVE_SPEC_SCHEMA_READING_TIMER).time()) {
       addSchemaPropertiesHelper(path, hiveUnit);
     }
@@ -281,6 +280,7 @@ protected void addSchemaPropertiesHelper(Path path, HiveRegistrationUnit hiveUni
       schema = TypeInfoUtils.getTypeInfoFromObjectInspector(
           TypeDescriptionToObjectInspectorUtil.getObjectInspector(orcSchema));
     } else {
+      Preconditions.checkArgument(this.fs.getFileStatus(path).isDirectory(), path + " is not a directory.");
       schema = getSchemaFromLatestFile(path, this.fs);
     }
     if (schema instanceof StructTypeInfo) {

diff --git a/...in-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java b/...in-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java
@@ -456,12 +456,9 @@ private void schemaUpdateHelper(GobblinMetadataChangeEvent gmce, HiveSpec spec,
     //Force to set the schema even there is no schema literal defined in the spec
     String latestSchema = latestSchemaMap.get(tableKey);
     if (latestSchema != null) {
-      String tableSchema = spec.getTable().getSerDeProps().getProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName());
-      if (tableSchema == null || !tableSchema.equals(latestSchema)) {
-        spec.getTable().getSerDeProps()
-            .setProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), latestSchemaMap.get(tableKey));
-        HiveMetaStoreUtils.updateColumnsInfoIfNeeded(spec);
-      }
+      spec.getTable().getSerDeProps()
+          .setProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), latestSchema);
+      HiveMetaStoreUtils.updateColumnsInfoIfNeeded(spec);
     }
   }