From 1e1023c928e7e4651f11b611b2f5da506a49ca15 Mon Sep 17 00:00:00 2001 From: Carles Arnal Date: Thu, 25 Apr 2024 15:27:27 +0200 Subject: [PATCH] Initial json schema dereferencing in the server --- app/pom.xml | 1 + .../noprofile/serde/JsonSchemaSerdeTest.java | 187 ++++++++++++++++-- pom.xml | 6 + schema-util/json/pom.xml | 4 + .../dereference/JsonSchemaDereferencer.java | 22 ++- .../AvroArtifactTypeUtilProvider.java | 3 +- .../JsonArtifactTypeUtilProvider.java | 3 +- .../OpenApiArtifactTypeUtilProvider.java | 3 +- 8 files changed, 205 insertions(+), 24 deletions(-) diff --git a/app/pom.xml b/app/pom.xml index 6213724231..1c7948f75d 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -133,6 +133,7 @@ io.quarkus quarkus-smallrye-context-propagation + io.quarkus quarkus-rest-client diff --git a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java index 9bc0e8fc3a..8b11f0ec4b 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java @@ -96,7 +96,7 @@ public void testJsonSchemaSerde() throws Exception { Person person = new Person("Ales", "Justin", 23); try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); @@ -119,7 +119,8 @@ public void testJsonSchemaSerde() throws Exception { try { serializer.serialize(artifactId, new RecordHeaders(), person); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } serializer.setValidationEnabled(false); // disable validation @@ -129,7 +130,8 @@ public void testJsonSchemaSerde() throws Exception { try { deserializer.deserialize(artifactId, headers, bytes); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } } } @@ -142,7 +144,7 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception { Person person = new Person("Carles", "Arnal", 30); try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); @@ -167,7 +169,8 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception { try { serializer.serialize(artifactId, new RecordHeaders(), person); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } serializer.setValidationEnabled(false); // disable validation @@ -177,7 +180,8 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception { try { deserializer.deserialize(artifactId, headers, bytes); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } } } @@ -197,7 +201,7 @@ public void testJsonSchemaSerdeHeaders() throws Exception { Person person = new Person("Ales", "Justin", 23); try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); @@ -243,7 +247,7 @@ public void testJsonSchemaSerdeMagicByte() throws Exception { Person person = new Person("Ales", "Justin", 23); try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); @@ -282,14 +286,12 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { Assertions.assertNotNull(qualificationSchema); Assertions.assertNotNull(addressSchema); - String groupId = TestUtils.generateGroupId(); String cityArtifactId = generateArtifactId(); String qualificationsId = generateArtifactId(); String identifierArtifactId = generateArtifactId(); String addressId = generateArtifactId(); - final Integer cityDependencyGlobalId = createArtifact(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema)); this.waitForGlobalId(cityDependencyGlobalId); @@ -328,7 +330,8 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { String artifactId = generateArtifactId(); - final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), List.of(qualificationsReference, cityReference, identifierReference, addressReference)); + final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), + List.of(qualificationsReference, cityReference, identifierReference, addressReference)); this.waitForGlobalId(globalId); City city = new City("New York", 10001); @@ -336,7 +339,7 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList()); try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); @@ -360,7 +363,8 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { try { serializer.serialize(artifactId, new RecordHeaders(), citizen); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } citizen.setAge(23); @@ -370,9 +374,151 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { try { serializer.serialize(artifactId, new RecordHeaders(), citizen); Assertions.fail(); - } catch (Exception ignored) { } + catch (Exception ignored) { + } + + //invalid identifier present, should fail + identifier = new CitizenIdentifier(-1234356); + citizen.setIdentifier(identifier); + + city = new City("Kansas CIty", 22222); + citizen.setCity(city); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } + catch (Exception ignored) { + } + + //no identifier present, should pass + citizen.setIdentifier(null); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + + //valid qualification, should pass + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7), + new Qualification(UUID.randomUUID().toString(), 8))); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + + //invalid qualification, should fail + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7), + new Qualification(UUID.randomUUID().toString(), 8))); + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } + catch (Exception ignored) { + } + } + } + + @Test + public void testJsonSchemaSerdeWithReferencesDeserializerDereferenced() throws Exception { + InputStream citySchema = getClass().getResourceAsStream("/io/apicurio/registry/util/city.json"); + InputStream citizenSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/citizen.json"); + InputStream citizenIdentifier = getClass().getResourceAsStream("/io/apicurio/registry/util/citizenIdentifier.json"); + InputStream qualificationSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/qualification.json"); + + InputStream addressSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/sample.address.json"); + Assertions.assertNotNull(citizenSchema); + Assertions.assertNotNull(citySchema); + Assertions.assertNotNull(citizenIdentifier); + Assertions.assertNotNull(qualificationSchema); + Assertions.assertNotNull(addressSchema); + + String groupId = TestUtils.generateGroupId(); + String cityArtifactId = generateArtifactId(); + String qualificationsId = generateArtifactId(); + String identifierArtifactId = generateArtifactId(); + String addressId = generateArtifactId(); + + final Integer cityDependencyGlobalId = createArtifact(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema)); + this.waitForGlobalId(cityDependencyGlobalId); + + final Integer qualificationsGlobalId = createArtifact(groupId, qualificationsId, ArtifactType.JSON, IoUtil.toString(qualificationSchema)); + this.waitForGlobalId(qualificationsGlobalId); + + final ArtifactReference qualificationsReference = new ArtifactReference(); + qualificationsReference.setVersion("1"); + qualificationsReference.setGroupId(groupId); + qualificationsReference.setArtifactId(qualificationsId); + qualificationsReference.setName("qualification.json"); + + final Integer addressGlobalID = createArtifact(groupId, addressId, ArtifactType.JSON, IoUtil.toString(addressSchema)); + this.waitForGlobalId(addressGlobalID); + + final ArtifactReference addressReference = new ArtifactReference(); + addressReference.setVersion("1"); + addressReference.setGroupId(groupId); + addressReference.setArtifactId(addressId); + addressReference.setName("sample.address.json"); + + final ArtifactReference cityReference = new ArtifactReference(); + cityReference.setVersion("1"); + cityReference.setGroupId(groupId); + cityReference.setArtifactId(cityArtifactId); + cityReference.setName("city.json"); + + final Integer identifierDependencyGlobalId = createArtifact(groupId, identifierArtifactId, ArtifactType.JSON, IoUtil.toString(citizenIdentifier)); + this.waitForGlobalId(identifierDependencyGlobalId); + + final ArtifactReference identifierReference = new ArtifactReference(); + identifierReference.setVersion("1"); + identifierReference.setGroupId(groupId); + identifierReference.setArtifactId(identifierArtifactId); + identifierReference.setName("citizenIdentifier.json"); + + String artifactId = generateArtifactId(); + + final Integer globalId = createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), + List.of(qualificationsReference, cityReference, identifierReference, addressReference)); + this.waitForGlobalId(globalId); + + City city = new City("New York", 10001); + CitizenIdentifier identifier = new CitizenIdentifier(123456789); + Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList()); + + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + + Map config = new HashMap<>(); + config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); + config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); + serializer.configure(config, false); + + deserializer.configure(Map.of(SchemaResolverConfig.DESERIALIZER_DEREFERENCE_SCHEMA, "true"), false); + + Headers headers = new RecordHeaders(); + byte[] bytes = serializer.serialize(artifactId, headers, citizen); + + citizen = deserializer.deserialize(artifactId, headers, bytes); + + Assertions.assertEquals("Carles", citizen.getFirstName()); + Assertions.assertEquals("Arnal", citizen.getLastName()); + Assertions.assertEquals(23, citizen.getAge()); + Assertions.assertEquals("New York", citizen.getCity().getName()); + + citizen.setAge(-1); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } + catch (Exception ignored) { + } + + citizen.setAge(23); + city = new City("Kansas CIty", -31); + citizen.setCity(city); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } + catch (Exception ignored) { + } //invalid identifier present, should fail identifier = new CitizenIdentifier(-1234356); @@ -384,7 +530,8 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { try { serializer.serialize(artifactId, new RecordHeaders(), citizen); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } //no identifier present, should pass @@ -392,15 +539,18 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { serializer.serialize(artifactId, new RecordHeaders(), citizen); //valid qualification, should pass - citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7), new Qualification(UUID.randomUUID().toString(), 8))); + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), 7), + new Qualification(UUID.randomUUID().toString(), 8))); serializer.serialize(artifactId, new RecordHeaders(), citizen); //invalid qualification, should fail - citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7), new Qualification(UUID.randomUUID().toString(), 8))); + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), new Qualification(UUID.randomUUID().toString(), -7), + new Qualification(UUID.randomUUID().toString(), 8))); try { serializer.serialize(artifactId, new RecordHeaders(), citizen); Assertions.fail(); - } catch (Exception ignored) { + } + catch (Exception ignored) { } } } @@ -442,7 +592,6 @@ public void complexObjectValidation() throws Exception { ContentTypes.APPLICATION_CREATE_EXTENDED, null, null, phone, null); - final ArtifactReference addressReference = new ArtifactReference(); addressReference.setVersion(amdAddress.getVersion()); addressReference.setGroupId(amdAddress.getGroupId()); diff --git a/pom.xml b/pom.xml index 427d63edf6..b6f3891043 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,7 @@ 1.11.3 1.4.0 + 4.5.7 4.9.9 4.9.9 3.9.0 @@ -562,6 +563,11 @@ json-schema-validator ${json-schema-validator.version} + + io.vertx + vertx-json-schema + ${vertx-json-schema.version} + com.squareup.wire wire-schema diff --git a/schema-util/json/pom.xml b/schema-util/json/pom.xml index 38b9bc160f..ba8b20af9f 100644 --- a/schema-util/json/pom.xml +++ b/schema-util/json/pom.xml @@ -42,6 +42,10 @@ com.github.everit-org.json-schema org.everit.json.schema + + io.vertx + vertx-json-schema + com.fasterxml.jackson.datatype jackson-datatype-json-org diff --git a/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java b/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java index 189fc51e86..f391b7c012 100644 --- a/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java +++ b/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java @@ -26,7 +26,13 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; import io.apicurio.registry.content.ContentHandle; +import io.vertx.core.json.JsonObject; +import io.vertx.json.schema.JsonSchema; +import io.vertx.json.schema.JsonSchemaOptions; +import io.vertx.json.schema.SchemaRepository; +import io.vertx.json.schema.impl.JsonRef; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -52,7 +58,18 @@ public class JsonSchemaDereferencer implements ContentDereferencer { @Override public ContentHandle dereference(ContentHandle content, Map resolvedReferences) { - throw new DereferencingNotSupportedException("Content dereferencing is not supported for JSON Schema"); + SchemaRepository schemaRepository = SchemaRepository.create(new JsonSchemaOptions().setBaseUri("https://test.com")); + Map lookups = new HashMap<>(); + resolveReferences(resolvedReferences, lookups); + JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(content.content()), lookups); + return ContentHandle.create(JsonSchema.of(schemaRepository.resolve(resolvedSchema)).toString()); + } + + private void resolveReferences(Map resolvedReferences, Map lookups) { + resolvedReferences.forEach((referenceName, schema) -> { + JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(schema.content()), lookups); + lookups.put(referenceName, JsonSchema.of(resolvedSchema)); + }); } /** @@ -65,7 +82,8 @@ public ContentHandle rewriteReferences(ContentHandle content, Map