Skip to content

Commit

Permalink
7
Browse files Browse the repository at this point in the history
  • Loading branch information
@zy-kkk
zy-kkk committed Dec 10, 2024
1 parent 3c41a1e commit 690a196
Show file tree
Hide file tree
Showing 3 changed files with 294 additions and 19 deletions.
12 changes: 9 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,9 @@ public JdbcExternalCatalog(long catalogId, String name, String resource, Map<Str
throws DdlException {
super(catalogId, name, InitCatalogLog.Type.JDBC, comment);
this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props));
this.identifierMapping = new JdbcIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
this.identifierMapping = new JdbcIdentifierMapping(
(Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
Boolean.parseBoolean(getLowerCaseMetaNames()),
getMetaNamesMapping());
}

Expand Down Expand Up @@ -129,7 +131,9 @@ public void onRefresh(boolean invalidCache) {
jdbcClient.closeClient();
jdbcClient = null;
}
this.identifierMapping = new JdbcIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
this.identifierMapping = new JdbcIdentifierMapping(
(Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
Boolean.parseBoolean(getLowerCaseMetaNames()),
getMetaNamesMapping());
}

Expand Down Expand Up @@ -244,7 +248,9 @@ protected void initLocalObjectsImpl() {
public void gsonPostProcess() throws IOException {
super.gsonPostProcess();
if (this.identifierMapping == null) {
identifierMapping = new JdbcIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
identifierMapping = new JdbcIdentifierMapping(
(Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
Boolean.parseBoolean(getLowerCaseMetaNames()),
getMetaNamesMapping());
}
}
Expand Down
125 changes: 115 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ public class JdbcIdentifierMapping implements IdentifierMapping {
private static final Logger LOG = LogManager.getLogger(JdbcIdentifierMapping.class);

private final ObjectMapper mapper = new ObjectMapper();
private final boolean isLowerCaseTableNames;
private final boolean isLowerCaseMetaNames;
private final String metaNamesMapping;

public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) {
public JdbcIdentifierMapping(boolean isLowerCaseTableNames, boolean isLowerCaseMetaNames, String metaNamesMapping) {
this.isLowerCaseTableNames = isLowerCaseTableNames;
this.isLowerCaseMetaNames = isLowerCaseMetaNames;
this.metaNamesMapping = metaNamesMapping;
validateMappings();
Expand Down Expand Up @@ -125,9 +127,13 @@ private void validateMappings() {
try {
JsonNode rootNode = mapper.readTree(metaNamesMapping);

validateNode(rootNode.path("databases"), "databases", duplicateErrors);
validateNode(rootNode.path("tables"), "tables", duplicateErrors);
validateNode(rootNode.path("columns"), "columns", duplicateErrors);
Map<String, Set<String>> dbMappingCheck = Maps.newHashMap();
Map<String, Map<String, Set<String>>> tableMappingCheck = Maps.newHashMap();
Map<String, Map<String, Map<String, Set<String>>>> columnMappingCheck = Maps.newHashMap();

validateNode(rootNode.path("databases"), "databases", duplicateErrors, dbMappingCheck, null, null);
validateNode(rootNode.path("tables"), "tables", duplicateErrors, null, tableMappingCheck, null);
validateNode(rootNode.path("columns"), "columns", duplicateErrors, null, null, columnMappingCheck);

if (!duplicateErrors.isEmpty()) {
StringBuilder errorBuilder = new StringBuilder("Duplicate mapping found:\n");
Expand All @@ -142,38 +148,137 @@ private void validateMappings() {
}
}

private void validateNode(JsonNode nodes, String nodeType, Map<String, Set<String>> duplicateErrors) {
private void validateNode(JsonNode nodes,
String nodeType,
Map<String, Set<String>> duplicateErrors,
Map<String, Set<String>> dbMappingCheck,
Map<String, Map<String, Set<String>>> tableMappingCheck,
Map<String, Map<String, Map<String, Set<String>>>> columnMappingCheck) {
Map<String, String> mappingSet = Maps.newHashMap();
if (nodes.isArray()) {
for (JsonNode node : nodes) {
String remoteKey;
String remoteDb = null;
String remoteTbl = null;
switch (nodeType) {
case "databases":
remoteKey = node.path("remoteDatabase").asText();
break;
case "tables":
remoteDb = node.path("remoteDatabase").asText();
remoteKey = node.path("remoteTable").asText();
break;
case "columns":
remoteDb = node.path("remoteDatabase").asText();
remoteTbl = node.path("remoteTable").asText();
remoteKey = node.path("remoteColumn").asText();
break;
default:
throw new IllegalArgumentException("Unknown type: " + nodeType);
}
String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText());
String mapping = node.path("mapping").asText();

// Check for duplicate mappings
if (mappingSet.containsKey(mapping)) {
String existed = mappingSet.get(mapping);
if (existed != null) {
duplicateErrors
.computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet())
.add(String.format("Remote name: %s, duplicate mapping: %s (original: %s)",
remoteKey, mapping, mappingSet.get(mapping)));
remoteKey, mapping, existed));
} else {
mappingSet.put(mapping, remoteKey);
}

switch (nodeType) {
case "databases":
if (isLowerCaseMetaNames) {
checkCaseConflictForDatabase(mapping, dbMappingCheck, duplicateErrors, nodeType, remoteKey);
}
break;
case "tables":
if (isLowerCaseMetaNames || isLowerCaseTableNames) {
checkCaseConflictForTable(remoteDb, mapping, tableMappingCheck, duplicateErrors, nodeType,
remoteKey);
}
break;
case "columns":
checkCaseConflictForColumn(remoteDb, remoteTbl, mapping, columnMappingCheck, duplicateErrors,
nodeType, remoteKey);
break;
default:
break;
}
mappingSet.put(mapping, remoteKey);
}
}
}

private void checkCaseConflictForDatabase(String mapping,
Map<String, Set<String>> dbMappingCheck,
Map<String, Set<String>> duplicateErrors,
String nodeType,
String remoteKey) {
if (dbMappingCheck == null) {
return;
}
String lower = mapping.toLowerCase();
Set<String> variants = dbMappingCheck.computeIfAbsent(lower, k -> Sets.newLinkedHashSet());
if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) {
duplicateErrors
.computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet())
.add(String.format("Remote name: %s, case-only different mapping found: %s (existing variants: %s)",
remoteKey, mapping, variants));
}
variants.add(mapping);
}

private void checkCaseConflictForTable(String remoteDb,
String mapping,
Map<String, Map<String, Set<String>>> tableMappingCheck,
Map<String, Set<String>> duplicateErrors,
String nodeType,
String remoteKey) {
if (tableMappingCheck == null || remoteDb == null) {
return;
}
Map<String, Set<String>> dbMap = tableMappingCheck.computeIfAbsent(remoteDb, k -> Maps.newHashMap());
String lower = mapping.toLowerCase();
Set<String> variants = dbMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet());
if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) {
duplicateErrors
.computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet())
.add(String.format("Remote name: %s (database: %s), "
+ "case-only different mapping found: %s (existing variants: %s)",
remoteKey, remoteDb, mapping, variants));
}
variants.add(mapping);
}

private void checkCaseConflictForColumn(String remoteDb,
String remoteTbl,
String mapping,
Map<String, Map<String, Map<String, Set<String>>>> columnMappingCheck,
Map<String, Set<String>> duplicateErrors,
String nodeType,
String remoteKey) {
if (columnMappingCheck == null || remoteDb == null || remoteTbl == null) {
return;
}
Map<String, Map<String, Set<String>>> dbMap = columnMappingCheck.computeIfAbsent(remoteDb,
k -> Maps.newHashMap());
Map<String, Set<String>> tblMap = dbMap.computeIfAbsent(remoteTbl, k -> Maps.newHashMap());
String lower = mapping.toLowerCase();
Set<String> variants = tblMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet());

if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) {
duplicateErrors
.computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet())
.add(String.format(
"Remote name: %s (database: %s, table: %s), "
+ "case-only different mapping found: %s (existing variants: %s)",
remoteKey, remoteDb, remoteTbl, mapping, variants));
}
variants.add(mapping);
}

private String applyLowerCaseIfNeeded(String value) {
return isLowerCaseMetaNames ? value.toLowerCase() : value;
}
Expand Down
Loading

0 comments on commit 690a196

Please sign in to comment.