Skip to content

Commit

Permalink
add custom properties
Browse files Browse the repository at this point in the history
  • Loading branch information
@harshach
harshach committed Oct 17, 2024
1 parent aca92ae commit 89a4e37
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 39 deletions.
10 changes: 6 additions & 4 deletions ...metadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,8 @@ public Response getEntityTypeFields(
Fields fieldsParam = new Fields(Set.of("customProperties"));
Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, include, false);
SchemaFieldExtractor extractor = new SchemaFieldExtractor();
List<SchemaFieldExtractor.FieldDefinition> fieldsList = extractor.extractFields(typeEntity, entityType);
List<SchemaFieldExtractor.FieldDefinition> fieldsList =
extractor.extractFields(typeEntity, entityType);
return Response.ok(fieldsList).type(MediaType.APPLICATION_JSON).build();

} catch (Exception e) {
Expand All @@ -500,11 +501,12 @@ public Response getEntityTypeFields(
@GET
@Path("/customProperties")
@Produces(MediaType.APPLICATION_JSON)
public Response getAllCustomPropertiesByEntityType(@Context UriInfo uriInfo,
@Context SecurityContext securityContext) {
public Response getAllCustomPropertiesByEntityType(
@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
try {
SchemaFieldExtractor extractor = new SchemaFieldExtractor();
Map<String, List<SchemaFieldExtractor.FieldDefinition>> customPropertiesMap = extractor.extractAllCustomProperties(uriInfo, repository);
Map<String, List<SchemaFieldExtractor.FieldDefinition>> customPropertiesMap =
extractor.extractAllCustomProperties(uriInfo, repository);
return Response.ok(customPropertiesMap).build();
} catch (Exception e) {
LOG.error("Error fetching custom properties: {}", e.getMessage(), e);
Expand Down
77 changes: 42 additions & 35 deletions openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.everit.json.schema.*;
import org.everit.json.schema.loader.SchemaClient;
Expand All @@ -29,16 +30,13 @@
import org.openmetadata.sdk.exception.SchemaProcessingException;
import org.openmetadata.service.jdbi3.TypeRepository;

import javax.ws.rs.core.UriInfo;

@Slf4j
public class SchemaFieldExtractor {

public SchemaFieldExtractor() {

}
public SchemaFieldExtractor() {}

public List<FieldDefinition> extractFields(Type typeEntity, String entityType) throws SchemaProcessingException {
public List<FieldDefinition> extractFields(Type typeEntity, String entityType)
throws SchemaProcessingException {
String schemaPath = determineSchemaPath(entityType);
String schemaUri = "classpath:///" + schemaPath;
SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
Expand All @@ -47,25 +45,28 @@ public List<FieldDefinition> extractFields(Type typeEntity, String entityType) t
Set<String> processedFields = new HashSet<>();
Schema mainSchema = loadMainSchema(schemaPath, entityType, schemaUri, schemaClient);
extractFieldsFromSchema(mainSchema, "", fieldTypesMap, processingStack, processedFields);
addCustomProperties(typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
addCustomProperties(
typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
return convertMapToFieldList(fieldTypesMap);
}

public Map<String, List<FieldDefinition>> extractAllCustomProperties(UriInfo uriInfo, TypeRepository repository) {
public Map<String, List<FieldDefinition>> extractAllCustomProperties(
UriInfo uriInfo, TypeRepository repository) {
Map<String, List<FieldDefinition>> entityTypeToFields = new HashMap<>();
List<String> entityTypes = getAllEntityTypes();

for (String entityType : entityTypes) {
String schemaPath = determineSchemaPath(entityType);
String schemaUri = "classpath:///" + schemaPath;
SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties"));
Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false);
Map<String, String> fieldTypesMap = new LinkedHashMap<>();
Set<String> processedFields = new HashSet<>();
Deque<Schema> processingStack = new ArrayDeque<>();
addCustomProperties(typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
entityTypeToFields.put(entityType, convertMapToFieldList(fieldTypesMap));
String schemaPath = determineSchemaPath(entityType);
String schemaUri = "classpath:///" + schemaPath;
SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties"));
Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false);
Map<String, String> fieldTypesMap = new LinkedHashMap<>();
Set<String> processedFields = new HashSet<>();
Deque<Schema> processingStack = new ArrayDeque<>();
addCustomProperties(
typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
entityTypeToFields.put(entityType, convertMapToFieldList(fieldTypesMap));
}

return entityTypeToFields;
Expand All @@ -75,28 +76,31 @@ public static List<String> getAllEntityTypes() {
List<String> entityTypes = new ArrayList<>();
try {
String schemaDirectory = "json/schema/entity/";
Enumeration<URL> resources = SchemaFieldExtractor.class.getClassLoader().getResources(schemaDirectory);
Enumeration<URL> resources =
SchemaFieldExtractor.class.getClassLoader().getResources(schemaDirectory);
while (resources.hasMoreElements()) {
URL resourceUrl = resources.nextElement();
Path schemaDirPath = Paths.get(resourceUrl.toURI());

Files.walk(schemaDirPath)
.filter(Files::isRegularFile)
.filter(path -> path.toString().endsWith(".json"))
.forEach(path -> {
try (InputStream is = Files.newInputStream(path)) {
JSONObject jsonSchema = new JSONObject(new JSONTokener(is));
// Check if the schema is an entity type
if (isEntityType(jsonSchema)) {
String fileName = path.getFileName().toString();
String entityType = fileName.substring(0, fileName.length() - 5); // Remove ".json"
entityTypes.add(entityType);
LOG.debug("Found entity type: {}", entityType);
}
} catch (Exception e) {
LOG.error("Error reading schema file {}: {}", path, e.getMessage());
}
});
.forEach(
path -> {
try (InputStream is = Files.newInputStream(path)) {
JSONObject jsonSchema = new JSONObject(new JSONTokener(is));
// Check if the schema is an entity type
if (isEntityType(jsonSchema)) {
String fileName = path.getFileName().toString();
String entityType =
fileName.substring(0, fileName.length() - 5); // Remove ".json"
entityTypes.add(entityType);
LOG.debug("Found entity type: {}", entityType);
}
} catch (Exception e) {
LOG.error("Error reading schema file {}: {}", path, e.getMessage());
}
});
}
} catch (Exception e) {
LOG.error("Error scanning schema directory: {}", e.getMessage());
Expand All @@ -108,7 +112,9 @@ private static boolean isEntityType(JSONObject jsonSchema) {
return "@om-entity-type".equals(jsonSchema.optString("$comment"));
}

private Schema loadMainSchema(String schemaPath, String entityType, String schemaUri, SchemaClient schemaClient) throws SchemaProcessingException {
private Schema loadMainSchema(
String schemaPath, String entityType, String schemaUri, SchemaClient schemaClient)
throws SchemaProcessingException {
InputStream schemaInputStream = getClass().getClassLoader().getResourceAsStream(schemaPath);
if (schemaInputStream == null) {
LOG.error("Schema file not found at path: {}", schemaPath);
Expand Down Expand Up @@ -354,7 +360,8 @@ private Schema resolveSchemaByType(String typeName, String schemaUri, SchemaClie
}
}

private Schema loadSchema(String schemaPath, String schemaUri, SchemaClient schemaClient) throws SchemaProcessingException {
private Schema loadSchema(String schemaPath, String schemaUri, SchemaClient schemaClient)
throws SchemaProcessingException {
InputStream schemaInputStream = getClass().getClassLoader().getResourceAsStream(schemaPath);
if (schemaInputStream == null) {
LOG.error("Schema file not found at path: {}", schemaPath);
Expand Down

0 comments on commit 89a4e37

Please sign in to comment.