Skip to content

Commit

Permalink
Initial json schema dereferencing in the server
Browse files Browse the repository at this point in the history
  • Loading branch information
carlesarnal committed Apr 25, 2024
1 parent 0642184 commit 1e1023c
Show file tree
Hide file tree
Showing 8 changed files with 205 additions and 24 deletions.
1 change: 1 addition & 0 deletions app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-smallrye-context-propagation</artifactId>
</dependency>

<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-rest-client</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public void testJsonSchemaSerde() throws Exception {
Person person = new Person("Ales", "Justin", 23);

try (JsonSchemaKafkaSerializer<Person> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
Expand All @@ -119,7 +119,8 @@ public void testJsonSchemaSerde() throws Exception {
try {
serializer.serialize(artifactId, new RecordHeaders(), person);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}

serializer.setValidationEnabled(false); // disable validation
Expand All @@ -129,7 +130,8 @@ public void testJsonSchemaSerde() throws Exception {
try {
deserializer.deserialize(artifactId, headers, bytes);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}
}
}
Expand All @@ -142,7 +144,7 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception {
Person person = new Person("Carles", "Arnal", 30);

try (JsonSchemaKafkaSerializer<Person> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
Expand All @@ -167,7 +169,8 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception {
try {
serializer.serialize(artifactId, new RecordHeaders(), person);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}

serializer.setValidationEnabled(false); // disable validation
Expand All @@ -177,7 +180,8 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception {
try {
deserializer.deserialize(artifactId, headers, bytes);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}
}
}
Expand All @@ -197,7 +201,7 @@ public void testJsonSchemaSerdeHeaders() throws Exception {
Person person = new Person("Ales", "Justin", 23);

try (JsonSchemaKafkaSerializer<Person> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
Expand Down Expand Up @@ -243,7 +247,7 @@ public void testJsonSchemaSerdeMagicByte() throws Exception {
Person person = new Person("Ales", "Justin", 23);

try (JsonSchemaKafkaSerializer<Person> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {
Deserializer<Person> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
Expand Down Expand Up @@ -282,14 +286,12 @@ public void testJsonSchemaSerdeWithReferences() throws Exception {
Assertions.assertNotNull(qualificationSchema);
Assertions.assertNotNull(addressSchema);


String groupId = TestUtils.generateGroupId();
String cityArtifactId = generateArtifactId();
String qualificationsId = generateArtifactId();
String identifierArtifactId = generateArtifactId();
String addressId = generateArtifactId();


final Integer cityDependencyGlobalId = createArtifact(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema));
this.waitForGlobalId(cityDependencyGlobalId);

Expand Down Expand Up @@ -328,15 +330,16 @@ public void testJsonSchemaSerdeWithReferences() throws Exception {

String artifactId = generateArtifactId();

final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), List.of(qualificationsReference, cityReference, identifierReference, addressReference));
final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema),
List.of(qualificationsReference, cityReference, identifierReference, addressReference));
this.waitForGlobalId(globalId);

City city = new City("New York", 10001);
CitizenIdentifier identifier = new CitizenIdentifier(123456789);
Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList());

try (JsonSchemaKafkaSerializer<Citizen> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Citizen> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {
Deserializer<Citizen> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
Expand All @@ -360,7 +363,8 @@ public void testJsonSchemaSerdeWithReferences() throws Exception {
try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}

citizen.setAge(23);
Expand All @@ -370,9 +374,151 @@ public void testJsonSchemaSerdeWithReferences() throws Exception {
try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}

//invalid identifier present, should fail
identifier = new CitizenIdentifier(-1234356);
citizen.setIdentifier(identifier);

city = new City("Kansas CIty", 22222);
citizen.setCity(city);

try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
}
catch (Exception ignored) {
}

//no identifier present, should pass
citizen.setIdentifier(null);
serializer.serialize(artifactId, new RecordHeaders(), citizen);

//valid qualification, should pass
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7),
new Qualification(UUID.randomUUID().toString(), 8)));
serializer.serialize(artifactId, new RecordHeaders(), citizen);

//invalid qualification, should fail
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7),
new Qualification(UUID.randomUUID().toString(), 8)));
try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
}
catch (Exception ignored) {
}
}
}

@Test
public void testJsonSchemaSerdeWithReferencesDeserializerDereferenced() throws Exception {
InputStream citySchema = getClass().getResourceAsStream("/io/apicurio/registry/util/city.json");
InputStream citizenSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/citizen.json");
InputStream citizenIdentifier = getClass().getResourceAsStream("/io/apicurio/registry/util/citizenIdentifier.json");
InputStream qualificationSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/qualification.json");

InputStream addressSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/sample.address.json");

Assertions.assertNotNull(citizenSchema);
Assertions.assertNotNull(citySchema);
Assertions.assertNotNull(citizenIdentifier);
Assertions.assertNotNull(qualificationSchema);
Assertions.assertNotNull(addressSchema);

String groupId = TestUtils.generateGroupId();
String cityArtifactId = generateArtifactId();
String qualificationsId = generateArtifactId();
String identifierArtifactId = generateArtifactId();
String addressId = generateArtifactId();

final Integer cityDependencyGlobalId = createArtifact(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema));
this.waitForGlobalId(cityDependencyGlobalId);

final Integer qualificationsGlobalId = createArtifact(groupId, qualificationsId, ArtifactType.JSON, IoUtil.toString(qualificationSchema));
this.waitForGlobalId(qualificationsGlobalId);

final ArtifactReference qualificationsReference = new ArtifactReference();
qualificationsReference.setVersion("1");
qualificationsReference.setGroupId(groupId);
qualificationsReference.setArtifactId(qualificationsId);
qualificationsReference.setName("qualification.json");

final Integer addressGlobalID = createArtifact(groupId, addressId, ArtifactType.JSON, IoUtil.toString(addressSchema));
this.waitForGlobalId(addressGlobalID);

final ArtifactReference addressReference = new ArtifactReference();
addressReference.setVersion("1");
addressReference.setGroupId(groupId);
addressReference.setArtifactId(addressId);
addressReference.setName("sample.address.json");

final ArtifactReference cityReference = new ArtifactReference();
cityReference.setVersion("1");
cityReference.setGroupId(groupId);
cityReference.setArtifactId(cityArtifactId);
cityReference.setName("city.json");

final Integer identifierDependencyGlobalId = createArtifact(groupId, identifierArtifactId, ArtifactType.JSON, IoUtil.toString(citizenIdentifier));
this.waitForGlobalId(identifierDependencyGlobalId);

final ArtifactReference identifierReference = new ArtifactReference();
identifierReference.setVersion("1");
identifierReference.setGroupId(groupId);
identifierReference.setArtifactId(identifierArtifactId);
identifierReference.setName("citizenIdentifier.json");

String artifactId = generateArtifactId();

final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema),
List.of(qualificationsReference, cityReference, identifierReference, addressReference));
this.waitForGlobalId(globalId);

City city = new City("New York", 10001);
CitizenIdentifier identifier = new CitizenIdentifier(123456789);
Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList());

try (JsonSchemaKafkaSerializer<Citizen> serializer = new JsonSchemaKafkaSerializer<>(restClient, true);
Deserializer<Citizen> deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) {

Map<String, Object> config = new HashMap<>();
config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId);
config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName());
serializer.configure(config, false);

deserializer.configure(Map.of(SchemaResolverConfig.DESERIALIZER_DEREFERENCE_SCHEMA, "true"), false);

Headers headers = new RecordHeaders();
byte[] bytes = serializer.serialize(artifactId, headers, citizen);

citizen = deserializer.deserialize(artifactId, headers, bytes);

Assertions.assertEquals("Carles", citizen.getFirstName());
Assertions.assertEquals("Arnal", citizen.getLastName());
Assertions.assertEquals(23, citizen.getAge());
Assertions.assertEquals("New York", citizen.getCity().getName());

citizen.setAge(-1);

try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
}
catch (Exception ignored) {
}

citizen.setAge(23);
city = new City("Kansas CIty", -31);
citizen.setCity(city);

try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
}
catch (Exception ignored) {
}

//invalid identifier present, should fail
identifier = new CitizenIdentifier(-1234356);
Expand All @@ -384,23 +530,27 @@ public void testJsonSchemaSerdeWithReferences() throws Exception {
try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}

//no identifier present, should pass
citizen.setIdentifier(null);
serializer.serialize(artifactId, new RecordHeaders(), citizen);

//valid qualification, should pass
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7), new Qualification(UUID.randomUUID().toString(), 8)));
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7),
new Qualification(UUID.randomUUID().toString(), 8)));
serializer.serialize(artifactId, new RecordHeaders(), citizen);

//invalid qualification, should fail
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7), new Qualification(UUID.randomUUID().toString(), 8)));
citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7),
new Qualification(UUID.randomUUID().toString(), 8)));
try {
serializer.serialize(artifactId, new RecordHeaders(), citizen);
Assertions.fail();
} catch (Exception ignored) {
}
catch (Exception ignored) {
}
}
}
Expand Down Expand Up @@ -442,7 +592,6 @@ public void complexObjectValidation() throws Exception {
ContentTypes.APPLICATION_CREATE_EXTENDED, null, null, phone,
null);


final ArtifactReference addressReference = new ArtifactReference();
addressReference.setVersion(amdAddress.getVersion());
addressReference.setGroupId(amdAddress.getGroupId());
Expand Down
6 changes: 6 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
<!-- Schema types -->
<avro.version>1.11.3</avro.version>
<json-schema-validator.version>1.4.0</json-schema-validator.version>
<vertx-json-schema.version>4.5.7</vertx-json-schema.version>
<wire-schema.version>4.9.9</wire-schema.version>
<wire-compiler.version>4.9.9</wire-compiler.version>
<okio-jvm.version>3.9.0</okio-jvm.version>
Expand Down Expand Up @@ -562,6 +563,11 @@
<artifactId>json-schema-validator</artifactId>
<version>${json-schema-validator.version}</version>
</dependency>
<dependency>
<groupId>io.vertx</groupId>
<artifactId>vertx-json-schema</artifactId>
<version>${vertx-json-schema.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.wire</groupId>
<artifactId>wire-schema</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions schema-util/json/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@
<groupId>com.github.everit-org.json-schema</groupId>
<artifactId>org.everit.json.schema</artifactId>
</dependency>
<dependency>
<groupId>io.vertx</groupId>
<artifactId>vertx-json-schema</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-json-org</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule;
import io.apicurio.registry.content.ContentHandle;
import io.vertx.core.json.JsonObject;
import io.vertx.json.schema.JsonSchema;
import io.vertx.json.schema.JsonSchemaOptions;
import io.vertx.json.schema.SchemaRepository;
import io.vertx.json.schema.impl.JsonRef;

import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

Expand All @@ -52,7 +58,18 @@ public class JsonSchemaDereferencer implements ContentDereferencer {

@Override
public ContentHandle dereference(ContentHandle content, Map<String, ContentHandle> resolvedReferences) {
throw new DereferencingNotSupportedException("Content dereferencing is not supported for JSON Schema");
SchemaRepository schemaRepository = SchemaRepository.create(new JsonSchemaOptions().setBaseUri("https://test.com"));
Map<String, JsonSchema> lookups = new HashMap<>();
resolveReferences(resolvedReferences, lookups);
JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(content.content()), lookups);
return ContentHandle.create(JsonSchema.of(schemaRepository.resolve(resolvedSchema)).toString());
}

private void resolveReferences(Map<String, ContentHandle> resolvedReferences, Map<String, JsonSchema> lookups) {
resolvedReferences.forEach((referenceName, schema) -> {
JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(schema.content()), lookups);
lookups.put(referenceName, JsonSchema.of(resolvedSchema));
});
}

/**
Expand All @@ -65,7 +82,8 @@ public ContentHandle rewriteReferences(ContentHandle content, Map<String, String
rewriteIn(tree, resolvedReferenceUrls);
String converted = objectMapper.writeValueAsString(objectMapper.treeToValue(tree, Object.class));
return ContentHandle.create(converted);
} catch (Exception e) {
}
catch (Exception e) {
return content;
}
}
Expand Down
Loading

0 comments on commit 1e1023c

Please sign in to comment.