diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 5095f07f2829d5d..1d7b785975c8f9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -83,7 +83,9 @@ public JdbcExternalCatalog(long catalogId, String name, String resource, Map> dbMappingCheck = Maps.newHashMap(); + Map>> tableMappingCheck = Maps.newHashMap(); + Map>>> columnMappingCheck = Maps.newHashMap(); + + validateNode(rootNode.path("databases"), "databases", duplicateErrors, dbMappingCheck, null, null); + validateNode(rootNode.path("tables"), "tables", duplicateErrors, null, tableMappingCheck, null); + validateNode(rootNode.path("columns"), "columns", duplicateErrors, null, null, columnMappingCheck); if (!duplicateErrors.isEmpty()) { StringBuilder errorBuilder = new StringBuilder("Duplicate mapping found:\n"); @@ -142,38 +148,137 @@ private void validateMappings() { } } - private void validateNode(JsonNode nodes, String nodeType, Map> duplicateErrors) { + private void validateNode(JsonNode nodes, + String nodeType, + Map> duplicateErrors, + Map> dbMappingCheck, + Map>> tableMappingCheck, + Map>>> columnMappingCheck) { Map mappingSet = Maps.newHashMap(); if (nodes.isArray()) { for (JsonNode node : nodes) { String remoteKey; + String remoteDb = null; + String remoteTbl = null; switch (nodeType) { case "databases": remoteKey = node.path("remoteDatabase").asText(); break; case "tables": + remoteDb = node.path("remoteDatabase").asText(); remoteKey = node.path("remoteTable").asText(); break; case "columns": + remoteDb = node.path("remoteDatabase").asText(); + remoteTbl = node.path("remoteTable").asText(); remoteKey = node.path("remoteColumn").asText(); break; default: throw new IllegalArgumentException("Unknown type: " + nodeType); } - String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + String mapping = node.path("mapping").asText(); - // Check for duplicate mappings - if (mappingSet.containsKey(mapping)) { + String existed = mappingSet.get(mapping); + if (existed != null) { duplicateErrors .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) .add(String.format("Remote name: %s, duplicate mapping: %s (original: %s)", - remoteKey, mapping, mappingSet.get(mapping))); + remoteKey, mapping, existed)); + } else { + mappingSet.put(mapping, remoteKey); + } + + switch (nodeType) { + case "databases": + if (isLowerCaseMetaNames) { + checkCaseConflictForDatabase(mapping, dbMappingCheck, duplicateErrors, nodeType, remoteKey); + } + break; + case "tables": + if (isLowerCaseMetaNames || isLowerCaseTableNames) { + checkCaseConflictForTable(remoteDb, mapping, tableMappingCheck, duplicateErrors, nodeType, + remoteKey); + } + break; + case "columns": + checkCaseConflictForColumn(remoteDb, remoteTbl, mapping, columnMappingCheck, duplicateErrors, + nodeType, remoteKey); + break; + default: + break; } - mappingSet.put(mapping, remoteKey); } } } + private void checkCaseConflictForDatabase(String mapping, + Map> dbMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (dbMappingCheck == null) { + return; + } + String lower = mapping.toLowerCase(); + Set variants = dbMappingCheck.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, case-only different mapping found: %s (existing variants: %s)", + remoteKey, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForTable(String remoteDb, + String mapping, + Map>> tableMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (tableMappingCheck == null || remoteDb == null) { + return; + } + Map> dbMap = tableMappingCheck.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = dbMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s (database: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForColumn(String remoteDb, + String remoteTbl, + String mapping, + Map>>> columnMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (columnMappingCheck == null || remoteDb == null || remoteTbl == null) { + return; + } + Map>> dbMap = columnMappingCheck.computeIfAbsent(remoteDb, + k -> Maps.newHashMap()); + Map> tblMap = dbMap.computeIfAbsent(remoteTbl, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = tblMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format( + "Remote name: %s (database: %s, table: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, remoteTbl, mapping, variants)); + } + variants.add(mapping); + } + private String applyLowerCaseIfNeeded(String value) { return isLowerCaseMetaNames ? value.toLowerCase() : value; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java index 43f2d287e74bcdd..edf7ab4e29b12a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java @@ -25,6 +25,8 @@ public class JdbcIdentifierMappingTest { private String validJson; private String invalidJson; private String duplicateMappingJson; + private String columnConflictJson; + private String tableConflictJson; @Before public void setUp() { @@ -56,11 +58,25 @@ public void setUp() { + " {\"remoteDatabase\": \"DORIS_DUP\", \"mapping\": \"CONFLICT\"}\n" + " ]\n" + "}"; + + columnConflictJson = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C1\", \"mapping\": \"custom_col\"},\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C2\", \"mapping\": \"CUSTOM_COL\"}\n" + + " ]\n" + + "}"; + + tableConflictJson = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T1\", \"mapping\": \"custom_table\"},\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T2\", \"mapping\": \"CUSTOM_TABLE\"}\n" + + " ]\n" + + "}"; } @Test public void testIsLowerCaseMetaNamesTrue() { - JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, validJson); + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, validJson); String databaseName = mapping.fromRemoteDatabaseName("DORIS"); String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); @@ -73,7 +89,7 @@ public void testIsLowerCaseMetaNamesTrue() { @Test public void testIsLowerCaseMetaNamesFalse() { - JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, validJson); + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, validJson); String databaseName = mapping.fromRemoteDatabaseName("DORIS"); String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); @@ -86,13 +102,13 @@ public void testIsLowerCaseMetaNamesFalse() { @Test(expected = RuntimeException.class) public void testInvalidJson() { - new JdbcIdentifierMapping(true, invalidJson); + new JdbcIdentifierMapping(true, false, invalidJson); } @Test public void testDuplicateMappingWhenLowerCaseMetaNamesTrue() { try { - new JdbcIdentifierMapping(true, duplicateMappingJson); + new JdbcIdentifierMapping(false, true, duplicateMappingJson); Assert.fail("Expected RuntimeException due to duplicate mappings"); } catch (RuntimeException e) { Assert.assertTrue(e.getMessage().contains("Duplicate mapping found")); @@ -101,8 +117,7 @@ public void testDuplicateMappingWhenLowerCaseMetaNamesTrue() { @Test public void testDuplicateMappingWhenLowerCaseMetaNamesFalse() { - // Should not throw exception because case-sensitive mappings are allowed - JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, duplicateMappingJson); + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, duplicateMappingJson); String databaseName1 = mapping.fromRemoteDatabaseName("DORIS"); String databaseName2 = mapping.fromRemoteDatabaseName("DORIS_DUP"); @@ -110,4 +125,153 @@ public void testDuplicateMappingWhenLowerCaseMetaNamesFalse() { Assert.assertEquals("conflict", databaseName1); Assert.assertEquals("CONFLICT", databaseName2); } + + @Test + public void testColumnCaseConflictAlwaysChecked() { + try { + new JdbcIdentifierMapping(false, false, columnConflictJson); + Assert.fail("Expected RuntimeException due to column case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseAndLowerCaseTableNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, tableConflictJson); + String tableName1 = mapping.fromRemoteTableName("D2", "T1"); + String tableName2 = mapping.fromRemoteTableName("D2", "T2"); + Assert.assertEquals("custom_table", tableName1); + Assert.assertEquals("CUSTOM_TABLE", tableName2); + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(true, false, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseButLowerCaseTableNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_local", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_LOCAL", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForColumnWithoutLowerCaseMetaNames() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_CLOCAL", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } + + @Test + public void testUppercaseMappingForColumnWithLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_clocal", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } }