From 566ab1c0aae3adeada8056736a47696fb48c7f24 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Fri, 29 Sep 2023 16:01:23 -0700 Subject: [PATCH] Adds writers for SchemaDocument and types --- .../ionschema/writer/IonSchemaWriter.kt | 26 ++-- .../writer/internal/IonSchemaWriterV2_0.kt | 43 ++++++ .../writer/internal/TypeWriterV2_0.kt | 125 +++++++++++++++++ .../internal/VersionedIonSchemaWriter.kt | 28 ++++ .../ionschema/writer/IonSchemaWriterTest.kt | 36 +++++ .../amazon/ionschema/writer/WriterTests.kt | 128 ++++++++++++++++++ 6 files changed, 372 insertions(+), 14 deletions(-) create mode 100644 ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/IonSchemaWriterV2_0.kt create mode 100644 ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/TypeWriterV2_0.kt create mode 100644 ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/VersionedIonSchemaWriter.kt create mode 100644 ion-schema/src/test/kotlin/com/amazon/ionschema/writer/IonSchemaWriterTest.kt create mode 100644 ion-schema/src/test/kotlin/com/amazon/ionschema/writer/WriterTests.kt diff --git a/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/IonSchemaWriter.kt b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/IonSchemaWriter.kt index 76c4391..4c567c9 100644 --- a/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/IonSchemaWriter.kt +++ b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/IonSchemaWriter.kt @@ -4,28 +4,26 @@ package com.amazon.ionschema.writer import com.amazon.ion.IonWriter +import com.amazon.ionschema.IonSchemaVersion import com.amazon.ionschema.model.ExperimentalIonSchemaModel -import com.amazon.ionschema.model.NamedTypeDefinition import com.amazon.ionschema.model.SchemaDocument -import com.amazon.ionschema.model.TypeDefinition +import com.amazon.ionschema.writer.internal.IonSchemaWriterV2_0 +import com.amazon.ionschema.writer.internal.VersionedIonSchemaWriter /** * Writes Ion Schema model to an IonWriter. */ @ExperimentalIonSchemaModel -interface IonSchemaWriter { +object IonSchemaWriter { /** * Writes a [SchemaDocument]. */ - fun writeSchema(ionWriter: IonWriter, schemaDocument: SchemaDocument) - - /** - * Writes an orphaned [TypeDefinition]—that is an anonymous type definition that does not belong to any schema. - */ - fun writeType(ionWriter: IonWriter, typeDefinition: TypeDefinition) - - /** - * Writes a [NamedTypeDefinition]. - */ - fun writeNamedType(ionWriter: IonWriter, namedTypeDefinition: NamedTypeDefinition) + @JvmStatic + fun writeSchema(ionWriter: IonWriter, schemaDocument: SchemaDocument) { + val delegate: VersionedIonSchemaWriter = when (schemaDocument.ionSchemaVersion) { + IonSchemaVersion.v1_0 -> TODO("IonSchemaWriter does not support ISL 1.0") + IonSchemaVersion.v2_0 -> IonSchemaWriterV2_0 + } + delegate.writeSchema(ionWriter, schemaDocument) + } } diff --git a/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/IonSchemaWriterV2_0.kt b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/IonSchemaWriterV2_0.kt new file mode 100644 index 0000000..61db03a --- /dev/null +++ b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/IonSchemaWriterV2_0.kt @@ -0,0 +1,43 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionschema.writer.internal + +import com.amazon.ion.IonWriter +import com.amazon.ionschema.IonSchemaVersion +import com.amazon.ionschema.internal.util.islRequire +import com.amazon.ionschema.model.ExperimentalIonSchemaModel +import com.amazon.ionschema.model.NamedTypeDefinition +import com.amazon.ionschema.model.SchemaDocument +import com.amazon.ionschema.model.SchemaFooter +import com.amazon.ionschema.model.SchemaHeader +import com.amazon.ionschema.model.TypeDefinition + +@ExperimentalIonSchemaModel +internal object IonSchemaWriterV2_0 : VersionedIonSchemaWriter { + + private val headerWriter = HeaderWriter + private val typeWriter = TypeWriterV2_0 + private val footerWriter = FooterWriter + + override fun writeSchema(ionWriter: IonWriter, schemaDocument: SchemaDocument) { + islRequire(schemaDocument.ionSchemaVersion == IonSchemaVersion.v2_0) { "IonSchemaWriterV2_0 only supports ISL 2.0" } + ionWriter.writeSymbol("\$ion_schema_2_0") + for (item in schemaDocument.items) { + when (item) { + is SchemaHeader -> headerWriter.writeHeader(ionWriter, item) + is NamedTypeDefinition -> writeNamedType(ionWriter, item) + is SchemaFooter -> footerWriter.writeFooter(ionWriter, item) + is SchemaDocument.OpenContent -> item.value.writeTo(ionWriter) + } + } + } + + override fun writeType(ionWriter: IonWriter, typeDefinition: TypeDefinition) { + typeWriter.writeOrphanedTypeDefinition(ionWriter, typeDefinition) + } + + override fun writeNamedType(ionWriter: IonWriter, namedTypeDefinition: NamedTypeDefinition) { + typeWriter.writeNamedTypeDefinition(ionWriter, namedTypeDefinition) + } +} diff --git a/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/TypeWriterV2_0.kt b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/TypeWriterV2_0.kt new file mode 100644 index 0000000..b09f77b --- /dev/null +++ b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/TypeWriterV2_0.kt @@ -0,0 +1,125 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionschema.writer.internal + +import com.amazon.ion.IonWriter +import com.amazon.ionschema.IonSchemaVersion +import com.amazon.ionschema.model.Constraint +import com.amazon.ionschema.model.DiscreteIntRange +import com.amazon.ionschema.model.ExperimentalIonSchemaModel +import com.amazon.ionschema.model.NamedTypeDefinition +import com.amazon.ionschema.model.OpenContentFields +import com.amazon.ionschema.model.TypeArgument +import com.amazon.ionschema.model.TypeDefinition +import com.amazon.ionschema.model.VariablyOccurringTypeArgument +import com.amazon.ionschema.writer.internal.constraints.AnnotationsV2Writer +import com.amazon.ionschema.writer.internal.constraints.ContainsWriter +import com.amazon.ionschema.writer.internal.constraints.ElementWriter +import com.amazon.ionschema.writer.internal.constraints.ExponentWriter +import com.amazon.ionschema.writer.internal.constraints.FieldNamesWriter +import com.amazon.ionschema.writer.internal.constraints.FieldsWriter +import com.amazon.ionschema.writer.internal.constraints.Ieee754FloatWriter +import com.amazon.ionschema.writer.internal.constraints.LengthConstraintsWriter +import com.amazon.ionschema.writer.internal.constraints.LogicConstraintsWriter +import com.amazon.ionschema.writer.internal.constraints.OrderedElementsWriter +import com.amazon.ionschema.writer.internal.constraints.PrecisionWriter +import com.amazon.ionschema.writer.internal.constraints.RegexWriter +import com.amazon.ionschema.writer.internal.constraints.TimestampOffsetWriter +import com.amazon.ionschema.writer.internal.constraints.TimestampPrecisionWriter +import com.amazon.ionschema.writer.internal.constraints.ValidValuesWriter + +@ExperimentalIonSchemaModel +internal object TypeWriterV2_0 : TypeWriter { + private val constraintWriters = listOf( + AnnotationsV2Writer(this), + ContainsWriter, + ElementWriter(this, IonSchemaVersion.v2_0), + ExponentWriter, + FieldNamesWriter(this), + FieldsWriter(this, IonSchemaVersion.v2_0), + Ieee754FloatWriter, + LengthConstraintsWriter, + LogicConstraintsWriter(this), + OrderedElementsWriter(this), + PrecisionWriter, + RegexWriter, + TimestampOffsetWriter, + TimestampPrecisionWriter, + ValidValuesWriter, + ).flatMap { w -> w.supportedClasses.map { it to w } } + .toMap() + + override fun writeNamedTypeDefinition(ionWriter: IonWriter, namedTypeDefinition: NamedTypeDefinition) { + ionWriter.setTypeAnnotations("type") + ionWriter.writeStruct { + ionWriter.setFieldName("name") + ionWriter.writeSymbol(namedTypeDefinition.typeName) + writeConstraints(ionWriter, namedTypeDefinition.typeDefinition.constraints) + writeOpenContent(ionWriter, namedTypeDefinition.typeDefinition.openContent) + } + } + + override fun writeTypeArg(ionWriter: IonWriter, typeArg: TypeArgument) { + if (typeArg.nullability == TypeArgument.Nullability.OrNull) { + ionWriter.addTypeAnnotation("\$null_or") + } + + when (typeArg) { + is TypeArgument.Import -> ionWriter.writeStruct { + ionWriter.setFieldName("id") + ionWriter.writeString(typeArg.schemaId) + ionWriter.setFieldName("type") + ionWriter.writeSymbol(typeArg.typeName) + } + is TypeArgument.InlineType -> ionWriter.writeStruct { + writeConstraints(ionWriter, typeArg.typeDefinition.constraints) + writeOpenContent(ionWriter, typeArg.typeDefinition.openContent) + } + is TypeArgument.Reference -> ionWriter.writeSymbol(typeArg.typeName) + } + } + + override fun writeVariablyOccurringTypeArg(ionWriter: IonWriter, varTypeArg: VariablyOccurringTypeArgument, elideOccursValue: DiscreteIntRange) { + if (varTypeArg.occurs == elideOccursValue) { + writeTypeArg(ionWriter, varTypeArg.typeArg) + } else { + ionWriter.writeStruct { + setFieldName("occurs") + writeRange(varTypeArg.occurs) + if (varTypeArg.typeArg.nullability == TypeArgument.Nullability.None && varTypeArg.typeArg is TypeArgument.InlineType) { + writeConstraints(ionWriter, varTypeArg.typeArg.typeDefinition.constraints) + writeOpenContent(ionWriter, varTypeArg.typeArg.typeDefinition.openContent) + } else { + setFieldName("type") + writeTypeArg(ionWriter, varTypeArg.typeArg) + } + } + } + } + + /** + * Writes a type that exists outside the context of any schema. + */ + fun writeOrphanedTypeDefinition(ionWriter: IonWriter, typeDefinition: TypeDefinition) { + ionWriter.setTypeAnnotations("type") + ionWriter.writeStruct { + writeConstraints(ionWriter, typeDefinition.constraints) + writeOpenContent(ionWriter, typeDefinition.openContent) + } + } + + private fun writeOpenContent(ionWriter: IonWriter, openContentFields: OpenContentFields) { + for ((fieldName, fieldValue) in openContentFields) { + ionWriter.setFieldName(fieldName) + fieldValue.writeTo(ionWriter) + } + } + + private fun writeConstraints(ionWriter: IonWriter, constraints: Set) { + for (c in constraints) { + constraintWriters[c::class]?.writeTo(ionWriter, c) + ?: TODO("Constraint not supported in Ion Schema 2.0: ${c::class}") + } + } +} diff --git a/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/VersionedIonSchemaWriter.kt b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/VersionedIonSchemaWriter.kt new file mode 100644 index 0000000..3a14983 --- /dev/null +++ b/ion-schema/src/main/kotlin/com/amazon/ionschema/writer/internal/VersionedIonSchemaWriter.kt @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionschema.writer.internal + +import com.amazon.ion.IonWriter +import com.amazon.ionschema.model.ExperimentalIonSchemaModel +import com.amazon.ionschema.model.NamedTypeDefinition +import com.amazon.ionschema.model.SchemaDocument +import com.amazon.ionschema.model.TypeDefinition + +@OptIn(ExperimentalIonSchemaModel::class) +interface VersionedIonSchemaWriter { + /** + * Writes a [SchemaDocument]. + */ + fun writeSchema(ionWriter: IonWriter, schemaDocument: SchemaDocument) + + /** + * Writes an orphaned [TypeDefinition]—that is an anonymous type definition that does not belong to any schema. + */ + fun writeType(ionWriter: IonWriter, typeDefinition: TypeDefinition) + + /** + * Writes a [NamedTypeDefinition]. + */ + fun writeNamedType(ionWriter: IonWriter, namedTypeDefinition: NamedTypeDefinition) +} diff --git a/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/IonSchemaWriterTest.kt b/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/IonSchemaWriterTest.kt new file mode 100644 index 0000000..8f4a264 --- /dev/null +++ b/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/IonSchemaWriterTest.kt @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionschema.writer + +import com.amazon.ionschema.ION +import com.amazon.ionschema.IonSchemaVersion +import com.amazon.ionschema.assertEqualIon +import com.amazon.ionschema.model.ExperimentalIonSchemaModel +import com.amazon.ionschema.model.SchemaDocument +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +@OptIn(ExperimentalIonSchemaModel::class) +class IonSchemaWriterTest { + // The purpose of these tests is just to check that it delegates to the correct writer implementation. + // Testing the actual serialization is done in WriterTests.kt + + @Test + fun `IonSchemaWriter throw UnsupportedOperationException for ISL 1 0`() { + val writer = ION.newTextWriter(StringBuilder()) + val schema = SchemaDocument("schema.isl", IonSchemaVersion.v1_0, emptyList()) + assertThrows { + IonSchemaWriter.writeSchema(writer, schema) + } + } + + @Test + fun `IonSchemaWriter writes a schema document for ISL 2 0`() { + val schema = SchemaDocument("schema.isl", IonSchemaVersion.v2_0, emptyList()) + // Since there's no content added to the schema, we expect just a version marker + assertEqualIon("\$ion_schema_2_0 ") { + IonSchemaWriter.writeSchema(it, schema) + } + } +} diff --git a/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/WriterTests.kt b/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/WriterTests.kt new file mode 100644 index 0000000..c8e3342 --- /dev/null +++ b/ion-schema/src/test/kotlin/com/amazon/ionschema/writer/WriterTests.kt @@ -0,0 +1,128 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionschema.writer + +import com.amazon.ion.IonList +import com.amazon.ion.IonStruct +import com.amazon.ion.IonSymbol +import com.amazon.ion.system.IonSystemBuilder +import com.amazon.ion.system.IonTextWriterBuilder +import com.amazon.ionschema.IonSchemaTests +import com.amazon.ionschema.IonSchemaVersion +import com.amazon.ionschema.TestFactory +import com.amazon.ionschema.asDocument +import com.amazon.ionschema.getTextField +import com.amazon.ionschema.model.ExperimentalIonSchemaModel +import com.amazon.ionschema.reader.IonSchemaReader +import com.amazon.ionschema.reader.IonSchemaReaderV2_0 +import com.amazon.ionschema.writer.internal.IonSchemaWriterV2_0 +import com.amazon.ionschema.writer.internal.VersionedIonSchemaWriter +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.DynamicContainer +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.Nested +import java.io.File +import java.util.stream.Stream + +@ExperimentalIonSchemaModel +class WriterTests { + + @Nested + inner class IonSchema_2_0 : TestFactory by WriterTestsRunner( + version = IonSchemaVersion.v2_0, + reader = IonSchemaReaderV2_0(), + writer = IonSchemaWriterV2_0, + ) +} + +@ExperimentalIonSchemaModel +class WriterTestsRunner( + val version: IonSchemaVersion, + val reader: IonSchemaReader, + val writer: VersionedIonSchemaWriter, + additionalFileFilter: (File) -> Boolean = { true }, + private val testNameFilter: (String) -> Boolean = { true }, +) : TestFactory { + + companion object { + private val ION = IonSystemBuilder.standard().build() + } + + private val fileFilter: (File) -> Boolean = { it.path.endsWith(".isl") && additionalFileFilter(it) } + private val baseDir = IonSchemaTests.testDirectoryFor(version) + + override fun generateTests(): Iterable { + return baseDir.walk() + .filter { it.isFile } + .filter(fileFilter) + .map { generateTestCases(it) } + .asIterable() + } + + private fun generateTestCases(f: File): DynamicNode { + val relativeFile = f.relativeTo(baseDir) + val dg = ION.loader.load(f) + + val fileIslVersion = dg[0].takeIf { IonSchemaVersion.isVersionMarker(it) } + ?.let { IonSchemaVersion.fromIonSymbolOrNull(it as IonSymbol) } + ?.takeIf { it == IonSchemaVersion.v2_0 } + ?: IonSchemaVersion.v1_0 + + if (fileIslVersion != IonSchemaVersion.v2_0) return DynamicContainer.dynamicContainer(relativeFile.path, f.toURI(), Stream.empty()) + + val validSchemaCase = + DynamicTest.dynamicTest("[$relativeFile] writeSchema should write a schema document that is equivalent to what was read") { + val schema = reader.readSchemaOrThrow(dg) + val newDg = ION.newDatagram() + val ionWriter = ION.newWriter(newDg) + writer.writeSchema(ionWriter, schema) + val schema2 = reader.readSchemaOrThrow(newDg) + assertEquals(schema, schema2) + } + + val dynamicNodeTestCases = dg.mapNotNull { ion -> + if (ion !is IonStruct) return@mapNotNull null + + when { + ion.hasTypeAnnotation("type") -> { + val displayName = "[$relativeFile] writeNamedType '${ion.getTextField("name")}'" + DynamicTest.dynamicTest(displayName) { + val type = reader.readNamedTypeOrThrow(ion) + val stringBuilder = StringBuilder() + val ionWriter = IonTextWriterBuilder.standard().build(stringBuilder) + writer.writeNamedType(ionWriter, type) + println(stringBuilder) + val type2 = reader.readNamedTypeOrThrow(ION.singleValue(stringBuilder.toString())) + assertEquals(type, type2) + } + } + IonSchemaTests.isValidSchemasTestCase(ion) -> createSchemasTestCases("$relativeFile", ion) + else -> null + } + } + + return DynamicContainer.dynamicContainer( + relativeFile.path, + f.toURI(), + (dynamicNodeTestCases + validSchemaCase).stream().filter { testNameFilter(it.displayName) } + ) + } + + private fun createSchemasTestCases(schemaId: String, ion: IonStruct): DynamicNode { + val baseDescription = ion.getTextField("description") + val cases = (ion["valid_schemas"] as IonList).mapIndexed { i, it -> + DynamicTest.dynamicTest("[$schemaId] $baseDescription [$i]") { + val dg = it.asDocument() + val schema = reader.readSchemaOrThrow(dg) + val newDg = ION.newDatagram() + val ionWriter = ION.newWriter(newDg) + writer.writeSchema(ionWriter, schema) + val schema2 = reader.readSchemaOrThrow(newDg) + assertEquals(schema, schema2) + } + } + return DynamicContainer.dynamicContainer("[$schemaId] $baseDescription", cases) + } +}