diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c648b84..9b7f0b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,8 @@ jobs: strategy: matrix: java: [8, 11] + subproject: [hive2, hive3] + fail-fast: false steps: - uses: actions/checkout@v2 with: @@ -21,4 +23,4 @@ jobs: with: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew :serde:check + run: ./gradlew ${{ matrix.subproject }}:build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87b1a52..2ec9361 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ information to effectively respond to your bug report or contribution. We welcome you to use the GitHub issue tracker to report bugs or suggest features. -When filing an issue, please check [existing open](https://github.com/amzn/amazon-ion-hive-serde/issues), or [recently closed](https://github.com/amzn/amazon-ion-hive-serde/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +When filing an issue, please check [existing open](https://github.com/amazon-ion/ion-hive-serde/issues), or [recently closed](https://github.com/amazon-ion/ion-hive-serde/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: * A reproducible test case or series of steps @@ -41,7 +41,7 @@ GitHub provides additional document on [forking a repository](https://help.githu ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amzn/amazon-ion-hive-serde/labels/help%20wanted) issues is a great place to start. +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amazon-ion/ion-hive-serde/labels/help%20wanted) issues is a great place to start. ## Code of Conduct @@ -56,6 +56,6 @@ If you discover a potential security issue in this project we ask that you notif ## Licensing -See the [LICENSE](https://github.com/amzn/amazon-ion-hive-serde/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. +See the [LICENSE](https://github.com/amazon-ion/ion-hive-serde/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/README.md b/README.md index 25817bc..0915a7e 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,36 @@ - -### :warning: Please refer to `hive2` branch for ion-hive2-serde and `hive3` branch for ion-hive3-serde. - ## Amazon Ion Hive Serde -A Apache Hive SerDe (short for serializer/deserializer) for the Ion file format. +An Apache Hive SerDe (short for serializer/deserializer) for the Ion file format, with support for Hive 2 and Hive 3. -[![Build Status](https://travis-ci.org/amzn/ion-hive-serde.svg?branch=master)](https://travis-ci.org/amzn/ion-hive-serde) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive-serde/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive-serde) -[![Javadoc](https://javadoc-badge.appspot.com/com.amazon.ion/ion-hive-serde.svg?label=javadoc)](http://www.javadoc.io/doc/com.amazon.ion/ion-hive-serde) +[![Build Status](https://github.com/amazon-ion/ion-hive-serde/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/amazon-ion/ion-hive-serde/actions?query=branch%3Amaster) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive2-serde/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive2-serde) +[![javadoc.io](https://javadoc.io/badge2/com.amazon.ion/ion-hive2-serde/javadoc.io.svg)](http://www.javadoc.io/doc/com.amazon.ion/ion-hive2-serde) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive3-serde/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-hive3-serde) +[![javadoc.io](https://javadoc.io/badge2/com.amazon.ion/ion-hive3-serde/javadoc.io.svg)](http://www.javadoc.io/doc/com.amazon.ion/ion-hive3-serde) ### Features * Read data stored in Ion format both binary and text. * Supports all Ion types including nested data structures, see [Type mapping documentation](docs/type-mapping.md) for more information. -* Supports flattening of Ion documents through [path extraction](https://github.com/amzn/ion-java-path-extraction). +* Supports flattening of Ion documents through [path extraction](https://github.com/amazon-ion/ion-java-path-extraction). * Supports importing shared symbol tables and custom symbol table catalogs. * `IonInputFormat` and `IonOutputFormat` are able to handle both Ion binary and Ion text. * Configurable through [SerDe properties](docs/serde-properties.md). ### Installation -Download the latest `ion-hive-serde-all-.jar` from [https://github.com/amzn/ion-hive-serde/releases] -and place the JARs into `hive/lib` or use `ADD JAR` in Hive. That jar contains the SerDe and all its dependencies. +Download the latest `ion-hive(2|3)-serde-all-.jar` from [https://github.com/amazon-ion/ion-hive-serde/releases]. (Some releases may have a slightly different jar name, such as `ion-hive(2|3)-serde--all.jar`.) -To build it locally run :`./gradlew :serde:singleJar` +To build it locally run :`./gradlew shadowJar` ### Building -Project is separated into two modules: -1. `serde`: with the SerDe code and unit tests. +Project is separated into modules: +1. `hive2`: with the SerDe code and unit tests for Hive 2. +1. `hive3`: with the SerDe code and unit tests for Hive 3. 1. `integration-tests`: integration tests using a dockerized hive installation. To build only the SerDe code: ``` -./gradlew :serde:build +./gradlew :hive2:build :hive3:build ``` To build the SerDe including integration tests: diff --git a/build.gradle b/build.gradle index 9b2e328..26e5577 100644 --- a/build.gradle +++ b/build.gradle @@ -1,83 +1,7 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -// sets up kotlin plugin -buildscript { - ext.kotlin_version = '[1.2.0,1.4.10)' - ext.ionjava_version = '[1.4.0,2.0.0)' - ext.pathextraction_version = '[1.2.0,2.0.0)' - ext.hive_version = '[2.3.0,2.4.0)' - ext.hadoop_version = '[2.7.0,2.8.0)' - - repositories { - mavenCentral() - } - - dependencies { - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - } -} +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 allprojects { - repositories { - mavenCentral() - } - - apply plugin: "kotlin" - - - configurations { - // to make compileOnly available to testing scope - testCompile.extendsFrom(compileOnly) - } - - dependencies { - // using kotlin to make tests less verbose - testCompile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" - testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version" - - testCompile "junit:junit:[4.0.0,5.0.0)" - testCompile "pl.pragmatists:JUnitParams:[1.1.0,1.2.0)" - } - - apply plugin: 'checkstyle' - checkstyle { - toolVersion = "8.18" - ignoreFailures = false - maxWarnings = 0 - maxErrors = 0 - configDir = file("$rootProject.projectDir/config/checkstyle") - } - tasks.withType(Checkstyle) { - reports { - xml.enabled = false - html.enabled = true - } - } -} - -// clean all subprojects -task cleanAll { - subprojects.each { - it.afterEvaluate { - def cleanTask = it.tasks.findByName('clean') - if (cleanTask) { - dependsOn(cleanTask) - } - } - } - - dependsOn(clean) + group = "com.amazon.ion" + version = "1.3.0-SNAPSHOT" } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 0000000..355563e --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +repositories { + gradlePluginPortal() + mavenCentral() +} + +plugins { + `kotlin-dsl` +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.9.22") + implementation("com.github.johnrengelman:shadow:8.1.1") +} diff --git a/buildSrc/src/main/kotlin/ion-hive-serde.conventions.gradle.kts b/buildSrc/src/main/kotlin/ion-hive-serde.conventions.gradle.kts new file mode 100644 index 0000000..86aee14 --- /dev/null +++ b/buildSrc/src/main/kotlin/ion-hive-serde.conventions.gradle.kts @@ -0,0 +1,57 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import org.jetbrains.kotlin.gradle.plugin.extraProperties +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import java.net.URI + +repositories { + mavenCentral() +} + +plugins { + kotlin("jvm") + id("checkstyle") +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 +} + +checkstyle { + toolVersion = "8.18" + // TODO: Fix checkstyle failures + isIgnoreFailures = false + maxWarnings = 0 + maxErrors = 0 +} + +tasks { + + withType().all { + // The release option is not available for the Java 8 compiler, but that's okay because we're targeting 8. + if (JavaVersion.current() != JavaVersion.VERSION_1_8) { + options.release.set(8) + } + } + + withType().all { + kotlinOptions { + jvmTarget = "1.8" + } + } + + withType().all { + useJUnitPlatform() + doLast { + logger.quiet("Test report written to: file://${reports.html.entryPoint}") + } + } + + withType().all { + reports { + xml.required.set(false) + html.required.set(true) + } + } +} diff --git a/buildSrc/src/main/kotlin/ion-hive-serde.dependencies.gradle.kts b/buildSrc/src/main/kotlin/ion-hive-serde.dependencies.gradle.kts new file mode 100644 index 0000000..4fa8192 --- /dev/null +++ b/buildSrc/src/main/kotlin/ion-hive-serde.dependencies.gradle.kts @@ -0,0 +1,57 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import org.jetbrains.kotlin.gradle.plugin.extraProperties +import java.net.URI + +/* + * This manages all of the dependencies by creating configurations for different groups of dependencies that belong + * together. These groups can be added to the standard configurations (e.g. `testImplementation`) by using the + * `extendFrom(...)` method in the `configurations` block. + */ + +// Dependencies that get bundled into the fat Jar +val bundled = configurations.create("bundled") +// Dependencies provided by Hive +val hive2Runtime = configurations.create("hive2Runtime") +val hive3Runtime = configurations.create("hive3Runtime") +// Test dependencies +val testFramework = configurations.create("testFramework") + +object Versions { + val hive2_version = "[2.3.0,2.4.0)" + val hive3_version = "[3.1.2,3.1.17)" + val kotlin_version = "1.9.22" + val ionjava_version = "[1.4.0,2.0.0)" + val pathextraction_version = "[1.2.0,2.0.0)" + val hadoop_version = "[2.7.0,2.8.0)" +} + +dependencies { + with(Versions) { + bundled("com.amazon.ion:ion-java:${ionjava_version}") + bundled("com.amazon.ion:ion-java-path-extraction:${pathextraction_version}") { + // ion-java-path-extraction uses Kotlin only for testing + exclude(group = "org.jetbrains.kotlin") + } + + // Provided by the Hive installation + hive2Runtime("org.apache.hive:hive-serde:${hive2_version}") + hive2Runtime("org.apache.hive:hive-exec:${hive2_version}") { isTransitive = false } + hive2Runtime("org.apache.hadoop:hadoop-common:${hadoop_version}") + hive2Runtime("org.apache.hadoop:hadoop-main:${hadoop_version}") + hive2Runtime("org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}") + + // Provided by the Hive installation + hive3Runtime("org.apache.hive:hive-serde:${hive3_version}") + hive3Runtime("org.apache.hive:hive-exec:${hive3_version}") { isTransitive = false } + hive3Runtime("org.apache.hadoop:hadoop-common:${hadoop_version}") + hive3Runtime("org.apache.hadoop:hadoop-main:${hadoop_version}") + hive3Runtime("org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}") + + testFramework("pl.pragmatists:JUnitParams:[1.1.0,1.2.0)") + testFramework("org.junit.jupiter:junit-jupiter:5.7.1") + testFramework("org.junit.vintage:junit-vintage-engine") + testFramework("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version") + } +} diff --git a/buildSrc/src/main/kotlin/ion-hive-serde.publishing.gradle.kts b/buildSrc/src/main/kotlin/ion-hive-serde.publishing.gradle.kts new file mode 100644 index 0000000..852be78 --- /dev/null +++ b/buildSrc/src/main/kotlin/ion-hive-serde.publishing.gradle.kts @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.tasks.bundling.Jar +import org.gradle.kotlin.dsl.create +import org.gradle.kotlin.dsl.get +import org.gradle.kotlin.dsl.kotlin +import org.jetbrains.kotlin.gradle.plugin.extraProperties +import java.net.URI + +repositories { + mavenCentral() +} + +plugins { + kotlin("jvm") + id("signing") + id("maven-publish") + id("com.github.johnrengelman.shadow") + id("ion-hive-serde.dependencies") +} + +tasks { + withType().all { + archiveBaseName.set("ion-${project.name}-serde") + } + + shadowJar { + configurations = listOf(project.configurations.getByName("bundled")) + } + + val testShadowJar = create("testShadowJar") { + group = "verification" + dependsOn(shadowJar) + classpath = shadowJar.get().outputs.files + + project.configurations.getByName("${project.name}Runtime") + + project.configurations.getByName("testFramework") + + project.sourceSets.test.get().output + } + + check { + dependsOn(testShadowJar) + } + + assemble { + dependsOn(shadowJar) + } +} + +val sourcesJar = tasks.create("sourcesJar") { + archiveClassifier.set("sources") + from(sourceSets["main"].java.srcDirs) +} + +val javadocJar = tasks.create("javadocJar") { + archiveClassifier.set("javadoc") + from(tasks.javadoc.get()) +} + +publishing { + publications { + publications.create("mavenJava") { + artifactId = tasks.jar.get().archiveBaseName.get() + artifact(tasks.jar) + artifact(sourcesJar) + artifact(javadocJar) + + pom { + name.set("Ion Hive Serde") + description.set("An Apache Hive SerDe (short for serializer/deserializer) for the Ion file format.") + url.set("https://github.com/amazon-ion/ion-hive-serde/") + + licenses { + license { + name.set("The Apache License, Version 2.0") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") + } + } + developers { + developer { + name.set("Amazon Ion Team") + email.set("ion-team@amazon.com") + organization.set("Amazon Ion") + organizationUrl.set("https://github.com/amazon-ion") + } + } + scm { + connection.set("scm:git:git@github.com:amazon-ion/ion-hive-serde.git") + developerConnection.set("scm:git:git@github.com:amazon-ion/ion-hive-serde.git") + url.set("git@github.com:amazon-ion/ion-hive-serde.git") + } + } + } + repositories.mavenCentral { + credentials { + username = properties["ossrhUsername"].toString() + password = properties["ossrhPassword"].toString() + } + url = URI.create("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2") + } + } +} + +signing { + sign(publishing.publications["mavenJava"]) +} diff --git a/config/checkstyle/copyright-header b/config/checkstyle/copyright-header index 281825e..cf1406c 100644 --- a/config/checkstyle/copyright-header +++ b/config/checkstyle/copyright-header @@ -1,14 +1,2 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ \ No newline at end of file +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 diff --git a/docs/serde-properties.md b/docs/serde-properties.md index fb3bdad..9c80021 100644 --- a/docs/serde-properties.md +++ b/docs/serde-properties.md @@ -26,7 +26,7 @@ WITH SERDEPROPERTIES ( ``` `OFFSET` is represented as `hh:mm`, examples: `01:00`, `+01:00`, `-09:30`, `Z` (UTC, same -as `00:00`), see [timestamp](http://amzn.github.io/ion-docs/docs/spec.html#timestamp) specification +as `00:00`), see [timestamp](http://amazon-ion.github.io/ion-docs/docs/spec.html#timestamp) specification for more details. ## Serialize null columns @@ -159,7 +159,7 @@ SELECT c1, c2 FROM myTable; ``` ## Path extractor -[Path extraction](https://github.com/amzn/ion-java-path-extraction) can be used to map between ion values and hive +[Path extraction](https://github.com/amazon-ion/ion-java-path-extraction) can be used to map between ion values and hive columns. Specification: @@ -350,10 +350,10 @@ Hive table ## Catalog Catalogs can be used by the SerDe to find any imported -[shared symbol tables](http://amzn.github.io/ion-docs/docs/symbols.html#shared-symbol-tables). +[shared symbol tables](http://amazon-ion.github.io/ion-docs/docs/symbols.html#shared-symbol-tables). Catalogs can be defined in one of three ways, in order or priority: -1. Class: implementation of [IonCatalog](https://github.com/amzn/ion-java/blob/master/src/com/amazon/ion/IonCatalog.java). +1. Class: implementation of [IonCatalog](https://github.com/amazon-ion/ion-java/blob/master/src/com/amazon/ion/IonCatalog.java). Must be added as a jar to hive in the same way the SerDe jar is included 1. Local File: shared symbol tables declared in a local file 1. URL: shared symbol tables resource URL diff --git a/docs/type-mapping.md b/docs/type-mapping.md index 39e3e2c..013b929 100644 --- a/docs/type-mapping.md +++ b/docs/type-mapping.md @@ -1,6 +1,6 @@ # Type mapping [Hive type system](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types) and the -[Ion type system](http://amzn.github.io/ion-docs/docs/spec.html) don't always map one to one so +[Ion type system](http://amazon-ion.github.io/ion-docs/docs/spec.html) don't always map one to one so some conversions must be made. For those [SerDe properties](serde-properties.md) can be used for fine tuning. **Type mapping from Ion types to Hive types during deserialization:** diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 29953ea..249e583 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 01011c9..a595206 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,21 +1,5 @@ -# -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - -#Wed Nov 28 12:57:14 PST 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip diff --git a/gradlew b/gradlew index cccdd3d..a69d9cb 100755 --- a/gradlew +++ b/gradlew @@ -1,78 +1,129 @@ -#!/usr/bin/env sh +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -81,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -89,84 +140,101 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" fi +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index f955316..53a6b23 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,84 +1,91 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/hive-common/build.gradle.kts b/hive-common/build.gradle.kts new file mode 100644 index 0000000..6a3fd17 --- /dev/null +++ b/hive-common/build.gradle.kts @@ -0,0 +1,50 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +plugins { + id("ion-hive-serde.dependencies") + id("ion-hive-serde.conventions") + id("java-test-fixtures") +} + +// Dependencies used by the test code that are not shared with anything else +val testImplementationHive2 by configurations.creating { + extendsFrom(configurations.testImplementation.get(), configurations.hive2Runtime.get()) +} +val testImplementationHive3 by configurations.creating { + extendsFrom(configurations.testImplementation.get(), configurations.hive3Runtime.get()) +} + +configurations { + implementation { extendsFrom(bundled.get()) } + // Have to pick one of the hive versions in order to get everything to compile. However, we will also compile + // using the other Hive version, and we will run the tests using both runtimes. + compileOnly { extendsFrom(hive3Runtime.get()) } + testCompileOnly { extendsFrom(hive3Runtime.get()) } + + // testImplementation also extendsFrom testFixturesImplementation + testFixturesImplementation { extendsFrom(configurations.testFramework.get()) } +} + +dependencies { + testImplementationHive2(testFixtures(project)) + testImplementationHive3(testFixtures(project)) +} + +tasks { + test { + classpath = testImplementationHive3 + project.sourceSets.test.get().output + } + + val compileJavaHive2 = create("compileJavaHive2") { + classpath = configurations.hive2Runtime.get() + configurations.bundled.get() + } + + val testHive2 = create("testHive2") { + dependsOn(compileJavaHive2) + classpath = testImplementationHive2 + project.sourceSets.test.get().output + useJUnitPlatform() + } + + check { dependsOn(testHive2) } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java b/hive-common/src/main/java/com/amazon/ionhiveserde/AbstractIonHiveSerDe.java similarity index 75% rename from serde/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/AbstractIonHiveSerDe.java index e41a1ef..f6fbda5 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/AbstractIonHiveSerDe.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde; @@ -25,7 +13,6 @@ import com.amazon.ionhiveserde.caseinsensitivedecorator.IonStructCaseInsensitiveDecorator; import com.amazon.ionhiveserde.configuration.IonEncoding; import com.amazon.ionhiveserde.configuration.SerDeProperties; -import com.amazon.ionhiveserde.objectinspectors.factories.IonObjectInspectorFactory; import com.amazon.ionhiveserde.serializers.TableSerializer; import com.amazon.ionpathextraction.PathExtractor; import java.io.ByteArrayOutputStream; @@ -42,9 +29,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; @@ -52,47 +37,48 @@ /** *

- * Hive SerDe for the Amazon Ion data format. + * Hive SerDe for the Amazon Ion data format. *

*

* For more information on Hive SerDes see wiki. *

*/ -public class IonHiveSerDe extends AbstractSerDe { +public abstract class AbstractIonHiveSerDe extends AbstractSerDe { private ObjectInspector objectInspector; - private SerDeProperties serDeProperties; + protected SerDeProperties serDeProperties; private SerDeStats stats; private TableSerializer serializer; private IonFactory ionFactory; + protected abstract TableSerializer getTableSerializer(SerDeProperties properties); + + protected abstract ObjectInspector getObjectInspectorForTable(SerDeProperties properties); + @Override - @SuppressWarnings("deprecation") // we are forced to override this constructor even though it's deprecated - public void initialize(final @Nullable Configuration conf, final Properties properties) throws SerDeException { + // we are forced to override this constructor even though it's deprecated + public final void initialize(final @Nullable Configuration conf, final Properties properties) { stats = new SerDeStats(); final List columnNames = readColumnNames(properties); final List columnTypes = readColumnTypes(properties); - final StructTypeInfo tableInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo( - columnNames, - columnTypes); serDeProperties = new SerDeProperties(properties, columnNames, columnTypes); ionFactory = new IonFactory(serDeProperties); - objectInspector = IonObjectInspectorFactory.objectInspectorForTable(tableInfo, serDeProperties); - serializer = new TableSerializer(columnNames, serDeProperties); + objectInspector = getObjectInspectorForTable(serDeProperties); + serializer = getTableSerializer(serDeProperties); } @Override - public Class getSerializedClass() { + public final Class getSerializedClass() { return serDeProperties.getEncoding() == IonEncoding.BINARY ? BytesWritable.class : Text.class; } @Override - public Writable serialize(final Object data, final ObjectInspector objectInspector) throws SerDeException { + public final Writable serialize(final Object data, final ObjectInspector objectInspector) throws SerDeException { if (objectInspector.getCategory() != STRUCT) { throw new SerDeException("Can only serialize struct types, got: " + objectInspector.getTypeName()); } @@ -111,7 +97,7 @@ public Writable serialize(final Object data, final ObjectInspector objectInspect } @Override - public Object deserialize(final Writable blob) throws SerDeException { + public final Object deserialize(final Writable blob) throws SerDeException { final byte[] bytes; final int length; @@ -142,7 +128,7 @@ public Object deserialize(final Writable blob) throws SerDeException { 1. We need a key-value store to carry column values 2. The top-level IonStruct as a context object carries the IonSystem which we use as a ValueFactory in the callbacks created in PathExtractionConfig - Refer to https://github.com/amzn/ion-hive-serde/issues/61. + Refer to https://github.com/amazon-ion/ion-hive-serde/issues/61. */ IonStruct struct = domFactory.newEmptyStruct(); if (!serDeProperties.pathExtractorCaseSensitivity()) { @@ -167,12 +153,12 @@ public Object deserialize(final Writable blob) throws SerDeException { } @Override - public SerDeStats getSerDeStats() { + public final SerDeStats getSerDeStats() { return stats; } @Override - public ObjectInspector getObjectInspector() throws SerDeException { + public final ObjectInspector getObjectInspector() throws SerDeException { return objectInspector; } diff --git a/serde/src/main/java/com/amazon/ionhiveserde/IonFactory.java b/hive-common/src/main/java/com/amazon/ionhiveserde/IonFactory.java similarity index 84% rename from serde/src/main/java/com/amazon/ionhiveserde/IonFactory.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/IonFactory.java index a3a56da..0d3ef30 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/IonFactory.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/IonFactory.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde; @@ -23,6 +11,7 @@ import com.amazon.ion.system.IonSystemBuilder; import com.amazon.ion.system.IonTextWriterBuilder; import com.amazon.ionhiveserde.configuration.BaseProperties; + import java.io.InputStream; import java.io.OutputStream; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java similarity index 56% rename from serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java index 115b956..a5351ff 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonCaseInsensitiveDecorator.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.caseinsensitivedecorator; @@ -22,11 +10,11 @@ public class IonCaseInsensitiveDecorator { /** * Wraps an IonValue in an IonContainerCaseInsensitiveDecorator if it's an ion container. - + * * @return a case insensitive decorator wrapped Ion Value. */ public static IonValue wrapValue(final IonValue v) { - if (v == null || v.isNullValue()) { + if (v == null) { return null; } diff --git a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java similarity index 89% rename from serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java index fb11486..d8275e1 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonContainerCaseInsensitiveDecorator.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.caseinsensitivedecorator; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java similarity index 90% rename from serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java index 5e6ae20..c62e71d 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonSequenceCaseInsensitiveDecorator.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.caseinsensitivedecorator; @@ -30,9 +18,6 @@ import org.jetbrains.annotations.NotNull; - - - public class IonSequenceCaseInsensitiveDecorator extends IonContainerCaseInsensitiveDecorator implements IonSequence { private final IonSequence ionSequence; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java similarity index 85% rename from serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java index cd92ab1..14243a0 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/caseinsensitivedecorator/IonStructCaseInsensitiveDecorator.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.caseinsensitivedecorator; @@ -21,10 +9,8 @@ import com.amazon.ion.SymbolToken; import com.amazon.ion.UnknownSymbolException; import com.amazon.ion.ValueFactory; - import java.util.Map; - public class IonStructCaseInsensitiveDecorator extends IonContainerCaseInsensitiveDecorator implements IonStruct { private final IonStruct ionStruct; @@ -92,7 +78,7 @@ public ValueFactory add(final String fieldName) { Argument `fieldName` is case sensitive so below method removes the desired value, which means it's possible that struct.containsKey(fieldName) returns true while struct.remove(fieldName) does nothing. - Refer to https://github.com/amzn/ion-hive-serde/issues/60. + Refer to https://github.com/amazon-ion/ion-hive-serde/issues/60. */ public IonValue remove(final String fieldName) { return IonCaseInsensitiveDecorator.wrapValue(ionStruct.remove(fieldName)); diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java similarity index 66% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java index 2804754..09c3554 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/BaseProperties.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java similarity index 88% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java index 893d785..c01171f 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/CatalogConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; @@ -22,6 +10,7 @@ import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.system.SimpleCatalog; import com.amazon.ionhiveserde.configuration.source.RawConfiguration; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java similarity index 59% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java index b43ef98..cd89a46 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/EncodingConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java similarity index 74% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java index 9dd4eef..00cd05a 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/FailOnOverflowConfig.java @@ -1,21 +1,10 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; import com.amazon.ionhiveserde.configuration.source.RawConfiguration; + import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java new file mode 100644 index 0000000..3904efd --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.configuration; + +import com.amazon.ionhiveserde.configuration.source.RawConfiguration; + +/** + * Encapsulates Ion related Hadoop job properties. + */ +public class HadoopProperties extends BaseProperties { + + /** + * Constructor. + * + * @param configuration raw configuration. + */ + public HadoopProperties(final RawConfiguration configuration) { + super(configuration); + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java similarity index 59% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java index 85507e8..e450e5d 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java new file mode 100644 index 0000000..048198d --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.configuration; + +/** + * Possible Ion encodings. + */ +public enum IonEncoding { + /** + * Ion binary. + */ + BINARY, + + /** + * Ion text. + */ + TEXT +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java similarity index 79% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java index 4b86985..77169a9 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/PathExtractionConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; @@ -21,6 +9,7 @@ import com.amazon.ionhiveserde.configuration.source.RawConfiguration; import com.amazon.ionpathextraction.PathExtractor; import com.amazon.ionpathextraction.PathExtractorBuilder; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,11 +58,8 @@ class PathExtractionConfig { final BiFunction callback = (ionReader, struct) -> { final IonValue ionValue = struct.getSystem().newValue(ionReader); - if (ionValue.isNullValue()) { - struct.put(columnName, null); // Hive can't handle IonNull - } else { - struct.put(columnName, ionValue); - } + // Hive can't handle IonNull, and we will filter all IonNull later in object inspectors + struct.put(columnName, ionValue); return 0; }; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java similarity index 83% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java index abfd5d4..b2194d4 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerDeProperties.java @@ -1,23 +1,11 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; import com.amazon.ion.IonStruct; import com.amazon.ion.IonType; -import com.amazon.ionhiveserde.IonHiveSerDe; +import com.amazon.ionhiveserde.AbstractIonHiveSerDe; import com.amazon.ionhiveserde.configuration.source.JavaPropertiesAdapter; import com.amazon.ionhiveserde.configuration.source.RawConfiguration; import com.amazon.ionpathextraction.PathExtractor; @@ -37,11 +25,14 @@ public class SerDeProperties extends BaseProperties { private final SerializeAsConfig serializeAsConfig; private final SerializeNullConfig serializeNullConfig; private final TimestampOffsetConfig timestampOffsetConfig; + private final List columnNames; + private final List columnTypes; /** * Constructor. * - * @param properties {@link Properties} passed to {@link IonHiveSerDe#initialize(Configuration, Properties)}. + * @param properties {@link Properties} passed to + * {@link AbstractIonHiveSerDe#initialize(Configuration, Properties)}. * @param columnNames table column names in the same order as types * @param columnTypes table column types in the same order as names */ @@ -62,6 +53,16 @@ private SerDeProperties(final RawConfiguration configuration, serializeAsConfig = new SerializeAsConfig(configuration, columnTypes); serializeNullConfig = new SerializeNullConfig(configuration); timestampOffsetConfig = new TimestampOffsetConfig(configuration); + this.columnNames = columnNames; + this.columnTypes = columnTypes; + } + + public List getColumnNames() { + return columnNames; + } + + public List getColumnTypes() { + return columnTypes; } /** diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java similarity index 89% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java index 33fd3e6..083557e 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeAsConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java similarity index 61% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java index a36a69e..fc22c49 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java new file mode 100644 index 0000000..e3bffc7 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.configuration; + +/** + * Possible strategies for the serialize_null property. + */ +public enum SerializeNullStrategy { + /** + * Omit nulls. + */ + OMIT, + + /** + * Serialize strongly typed nulls. + */ + TYPED, + + /** + * Serialize untyped nulls. + */ + UNTYPED; +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java similarity index 85% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java index 52b9a4b..bb811e8 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/TimestampOffsetConfig.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java similarity index 60% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java index 4d8a2d9..cb714ed 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/HadoopConfigurationAdapter.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration.source; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java similarity index 60% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java index 99ef926..32b793b 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/JavaPropertiesAdapter.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration.source; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java similarity index 61% rename from serde/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java index 26edd9f..2907487 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/configuration/source/RawConfiguration.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration.source; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java b/hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java similarity index 91% rename from serde/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java index 8f88494..f71befb 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonInputFormat.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.formats; @@ -25,7 +13,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java b/hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java similarity index 89% rename from serde/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java index 2a7c4f3..389eb94 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/formats/IonOutputFormat.java @@ -1,25 +1,11 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.formats; -import com.amazon.ionhiveserde.IonHiveSerDe; - +import com.amazon.ionhiveserde.AbstractIonHiveSerDe; import java.io.DataOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.Optional; import java.util.Properties; import org.apache.hadoop.fs.FSDataOutputStream; @@ -41,12 +27,12 @@ /** *

- * Output format used in conjunction with the {@link IonHiveSerDe}. Handles both the Ion + * Output format used in conjunction with the {@link AbstractIonHiveSerDe}. Handles both the Ion * text or Ion binary serialized by the SerDe *

* *

- * WARNING: Must be used with {@link IonHiveSerDe}. + * WARNING: Must be used with {@link AbstractIonHiveSerDe}. *

*/ public class IonOutputFormat extends FileOutputFormat implements HiveOutputFormat { diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java similarity index 60% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java index 16d00a8..555e710 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java similarity index 74% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java index 1e9491b..53919ef 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -25,7 +13,7 @@ * @param Primitive type. */ abstract class AbstractOverflowablePrimitiveObjectInspector extends - AbstractIonPrimitiveJavaObjectInspector { + AbstractIonPrimitiveJavaObjectInspector { private final boolean failOnOverflow; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java similarity index 69% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java index c0ab9f4..f95abe9 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java similarity index 68% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java index abbf356..4af86ab 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java similarity index 71% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java index 7a662f9..4671ac9 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -25,7 +13,7 @@ * Adapts an {@link IonFloat} for the float Hive type. */ public class IonFloatToFloatObjectInspector extends - AbstractOverflowablePrimitiveObjectInspector implements + AbstractOverflowablePrimitiveObjectInspector implements FloatObjectInspector { public IonFloatToFloatObjectInspector(final boolean failOnOverflow) { diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java similarity index 75% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java index 3200b00..29684c9 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java similarity index 75% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java index 5c5494d..2af1dea 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java similarity index 78% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java index eab2965..7461c07 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java similarity index 75% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java index ef2c0f7..8b1bb8d 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -26,7 +14,7 @@ * Adapts an {@link IonInt} for the tinyint Hive type. */ public class IonIntToTinyIntObjectInspector extends - AbstractOverflowablePrimitiveObjectInspector implements + AbstractOverflowablePrimitiveObjectInspector implements ByteObjectInspector { public static final int MIN_VALUE = -128; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java similarity index 67% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java index 65e8c3d..a3e3c75 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java similarity index 76% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java index 9c2e558..0a80851 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java similarity index 74% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java index 99ce2b4..6742313 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -21,6 +9,7 @@ import com.amazon.ion.IonSequence; import com.amazon.ion.IonValue; import java.util.List; +import java.util.stream.Collectors; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -54,7 +43,7 @@ public Object getListElement(final Object data, final int index) { return null; } - return sequence.get(index); + return IonUtil.handleNull(sequence.get(index)); } @Override @@ -73,7 +62,8 @@ public List getList(final Object data) { return null; } - return (IonSequence) data; + final IonSequence sequence = (IonSequence) data; + return sequence.stream().map(IonUtil::handleNull).collect(Collectors.toList()); } @Override diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java similarity index 68% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java index b73f99b..5b48995 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -22,6 +10,7 @@ import com.amazon.ion.IonValue; import java.util.HashMap; import java.util.Map; +import java.util.stream.StreamSupport; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -63,7 +52,7 @@ public Object getMapValueElement(final Object data, final Object key) { final IonStruct struct = (IonStruct) data; final IonSymbol symbol = (IonSymbol) key; - return struct.get(symbol.stringValue()); + return IonUtil.handleNull(struct.get(symbol.stringValue())); } @Override @@ -74,16 +63,8 @@ public Object getMapValueElement(final Object data, final Object key) { final IonStruct struct = (IonStruct) data; - // sets the initial size of the map to avoid growing as it's immutable while using the default HashMap load - // factor to maintain the same collision performance - final int size = (int) Math.ceil(struct.size() / DEFAULT_LOAD_FACTOR); - - final Map map = new HashMap<>(size, DEFAULT_LOAD_FACTOR); - for (IonValue v : struct) { - map.put(v.getFieldName(), v); - } - - return map; + return StreamSupport.stream(struct.spliterator(), false) + .collect(HashMap::new, (m,v) -> m.put(v.getFieldName(), IonUtil.handleNull(v)), HashMap::putAll); } @Override diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java similarity index 83% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java index 45f0f1c..50b6045 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -23,6 +11,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -84,7 +74,7 @@ public Object getStructFieldData(final Object data, final StructField fieldRef) final IonStruct struct = (IonStruct) data; - return struct.get(fieldRef.getFieldName()); + return IonUtil.handleNull(struct.get(fieldRef.getFieldName())); } @Override @@ -95,12 +85,8 @@ public List getStructFieldsDataAsList(final Object data) { final IonStruct struct = (IonStruct) data; - List list = new ArrayList<>(struct.size()); - for (IonValue v : struct) { - list.add(v); - } - - return list; + return StreamSupport.stream(struct.spliterator(), false).map(IonUtil::handleNull) + .collect(Collectors.toList()); } @Override diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java similarity index 76% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java index d24b76d..51385c6 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java similarity index 66% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java index 755b027..0931733 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java similarity index 76% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java index 40388a7..ba121fe 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java similarity index 82% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java index 2924365..53d2201 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java new file mode 100644 index 0000000..04d4970 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java @@ -0,0 +1,20 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors; + +import com.amazon.ion.IonValue; + +/** + * Utility methods for Ion values. + */ +public class IonUtil { + + static boolean isIonNull(final IonValue ionValue) { + return ionValue == null || ionValue.isNullValue(); + } + + public static T handleNull(final T value) { + return isIonNull(value) ? null : value; + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java similarity index 66% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java index 8ba3099..72c0f42 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; @@ -23,7 +11,12 @@ import org.apache.hadoop.io.Text; /** - * Adapts an {@link IonText} for the string Hive type. + * Adapts an {@link IonValue} for the string Hive type. An {@link IonText} value + * will be represented by its content, while a non-text {@link IonValue} will be + * represented by an Ion textual representation. E.g. the Ion value + * {@code "bar"} is an Ion string and will map to {@code "bar"}, whereas the + * Ion value {@code { foo: "bar" }} is an Ion struct and would map to + * something like {@code "{foo: \"bar\"}" } . */ public class IonValueToStringObjectInspector extends AbstractIonPrimitiveJavaObjectInspector implements StringObjectInspector { diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java new file mode 100644 index 0000000..0aa53e8 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors; + +/** + * Validates a {@link String} length, throwing if the string is longer than expected. + */ +class TextMaxLengthValidator { + + /** + * Validates text size. + * + * @throws IllegalArgumentException if the string is longer than max length + */ + void validate(final String text, final int maxLength) { + if (text.length() > maxLength) { + throw new IllegalArgumentException( + "text is longer than allowed. Size: " + text.length() + ", max: " + maxLength); + } + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java similarity index 89% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java index 788b6fd..af53288 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/factories/IonObjectInspectorFactory.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors.factories; @@ -31,14 +19,13 @@ import com.amazon.ionhiveserde.objectinspectors.IonTextToCharObjectInspector; import com.amazon.ionhiveserde.objectinspectors.IonTextToStringObjectInspector; import com.amazon.ionhiveserde.objectinspectors.IonTextToVarcharObjectInspector; -import com.amazon.ionhiveserde.objectinspectors.IonTimestampToDateObjectInspector; -import com.amazon.ionhiveserde.objectinspectors.IonTimestampToTimestampObjectInspector; import com.amazon.ionhiveserde.objectinspectors.IonUnionObjectInspector; import com.amazon.ionhiveserde.objectinspectors.IonValueToStringObjectInspector; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -84,13 +71,21 @@ public class IonObjectInspectorFactory { new IonTextToStringObjectInspector(); private static final IonLobToBinaryObjectInspector LOB_TO_BINARY_OBJECT_INSPECTOR = new IonLobToBinaryObjectInspector(); - private static final IonTimestampToDateObjectInspector TIMESTAMP_TO_DATE_OBJECT_INSPECTOR = - new IonTimestampToDateObjectInspector(); - private static final IonTimestampToTimestampObjectInspector TIMESTAMP_TO_TIMESTAMP_OBJECT_INSPECTOR = - new IonTimestampToTimestampObjectInspector(); private static final IonValueToStringObjectInspector ION_VALUE_TO_STRING_OBJECT_INSPECTOR = new IonValueToStringObjectInspector(); + // Injected object inspectors + private final TimestampObjectInspector timestampToTimestampObjectInspector; + private final DateObjectInspector timestampToDateObjectInspector; + + public IonObjectInspectorFactory( + final DateObjectInspector timestampToDateObjectInspector, + final TimestampObjectInspector timestampToTimestampObjectInspector + ) { + this.timestampToDateObjectInspector = timestampToDateObjectInspector; + this.timestampToTimestampObjectInspector = timestampToTimestampObjectInspector; + } + /** * Creates an object inspector for the table correctly configured. * @@ -98,7 +93,7 @@ public class IonObjectInspectorFactory { * @param serDeProperties SerDe properties to configure the object inspector. * @return configured object inspector. */ - public static ObjectInspector objectInspectorForTable(final StructTypeInfo structTypeInfo, + public ObjectInspector objectInspectorForTable(final StructTypeInfo structTypeInfo, final SerDeProperties serDeProperties) { final List fieldObjectInspectors = new ArrayList<>(); @@ -125,7 +120,7 @@ public static ObjectInspector objectInspectorForTable(final StructTypeInfo struc * @param serDeProperties SerDe properties to configure the object inspector. * @return configured object inspector. */ - private static ObjectInspector objectInspectorForField(final TypeInfo typeInfo, + private ObjectInspector objectInspectorForField(final TypeInfo typeInfo, final String fieldName, final SerDeProperties serDeProperties) { final boolean failOnOverflow = serDeProperties.failOnOverflowFor(fieldName); @@ -199,11 +194,11 @@ private static ObjectInspector objectInspectorForField(final TypeInfo typeInfo, break; case DATE: - objectInspector = TIMESTAMP_TO_DATE_OBJECT_INSPECTOR; + objectInspector = timestampToDateObjectInspector; break; case TIMESTAMP: - objectInspector = TIMESTAMP_TO_TIMESTAMP_OBJECT_INSPECTOR; + objectInspector = timestampToTimestampObjectInspector; break; case VOID: diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java new file mode 100644 index 0000000..be4de51 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java @@ -0,0 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* + * ObjectInspectors are adapters from Ion types into Hive types. This package contains one ObjectInspector per + * valid Ion to Hive type combination + */ +package com.amazon.ionhiveserde.objectinspectors; + diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java similarity index 71% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java index 63c5a97..ce27077 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractLobSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java similarity index 85% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java index 6850964..ec485db 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractStructSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; @@ -31,10 +19,14 @@ * Base class for struct serializers. */ abstract class AbstractStructSerializer implements IonSerializer { - + private final IonSerializerFactory ionSerializerFactory; private final SerDeProperties properties; - AbstractStructSerializer(final SerDeProperties properties) { + AbstractStructSerializer( + final IonSerializerFactory ionSerializerFactory, + final SerDeProperties properties + ) { + this.ionSerializerFactory = ionSerializerFactory; this.properties = properties; } @@ -68,7 +60,7 @@ private void serializeMap(final IonWriter writer, for (Map.Entry entry : objectInspector.getMap(data).entrySet()) { final String key = keyObjectInspector.getPrimitiveJavaObject(entry.getKey()); - final IonSerializer ionSerializer = IonSerializerFactory.serializerForObjectInspector( + final IonSerializer ionSerializer = ionSerializerFactory.serializerForObjectInspector( valueObjectInspector, properties); @@ -85,7 +77,7 @@ private void serializeStruct(final IonWriter writer, writer.stepIn(IonType.STRUCT); for (StructField field : objectInspector.getAllStructFieldRefs()) { - final IonSerializer ionSerializer = IonSerializerFactory.serializerForObjectInspector( + final IonSerializer ionSerializer = ionSerializerFactory.serializerForObjectInspector( field.getFieldObjectInspector(), properties); diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java similarity index 79% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java index 7fbecf8..4f8aece 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/AbstractTextSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java new file mode 100644 index 0000000..11fb420 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +/** + * Serializer for Ion blob. + */ +class BlobSerializer extends AbstractLobSerializer { + + @Override + protected void writeValue(final IonWriter writer, final byte[] bytes) throws IOException { + writer.writeBlob(bytes); + } + + @Override + public IonType getIonType() { + return IonType.BLOB; + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java similarity index 68% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java index aeba195..f78f532 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/BoolSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java new file mode 100644 index 0000000..e4def76 --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +/** + * Serializer for clob. + */ +class ClobSerializer extends AbstractLobSerializer { + + @Override + public IonType getIonType() { + return IonType.CLOB; + } + + @Override + protected void writeValue(final IonWriter writer, final byte[] bytes) throws IOException { + writer.writeClob(bytes); + } +} diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java new file mode 100644 index 0000000..5e5246c --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ionhiveserde.configuration.SerDeProperties; + +/** + * Struct serializer. + */ +class ColumnStructSerializer extends AbstractStructSerializer { + + ColumnStructSerializer(final IonSerializerFactory ionSerializerFactory, final SerDeProperties properties) { + super(ionSerializerFactory, properties); + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java similarity index 71% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java index 10a70ed..5dd365d 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/DecimalSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java similarity index 71% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java index 2aac77c..3e41ee7 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/FloatSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java similarity index 79% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java index c3b871c..0c0f927 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IntSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java similarity index 62% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java index e27b1c6..861fd61 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java similarity index 72% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java index 35101c0..6832956 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/IonSerializerFactory.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; @@ -25,11 +13,11 @@ /** * Factory for Ion serializers, reuses serializers when possible. */ -class IonSerializerFactory { +abstract class IonSerializerFactory { - private static final Map serializerByTypeCache; + private final Map serializerByTypeCache; - static { + { serializerByTypeCache = new HashMap<>(); serializerByTypeCache.put(IonType.BOOL, new BoolSerializer()); serializerByTypeCache.put(IonType.INT, new IntSerializer()); @@ -41,27 +29,29 @@ class IonSerializerFactory { serializerByTypeCache.put(IonType.BLOB, new BlobSerializer()); } + protected abstract IonSerializer newTimestampSerializer(SerDeProperties properties); + /** * Gets an Ion serializer based on an ion type and serde properties. */ - static IonSerializer serializerForIon(final IonType ionType, final SerDeProperties properties) { + IonSerializer serializerForIon(final IonType ionType, final SerDeProperties properties) { IonSerializer ionSerializer = null; switch (ionType) { case TIMESTAMP: - ionSerializer = new TimestampSerializer(properties.getTimestampOffsetInMinutes()); + ionSerializer = newTimestampSerializer(properties); break; case LIST: - ionSerializer = new SequenceSerializer(properties, IonType.LIST); + ionSerializer = new SequenceSerializer(this, properties, IonType.LIST); break; case SEXP: - ionSerializer = new SequenceSerializer(properties, IonType.SEXP); + ionSerializer = new SequenceSerializer(this, properties, IonType.SEXP); break; case STRUCT: - ionSerializer = new ColumnStructSerializer(properties); + ionSerializer = new ColumnStructSerializer(this, properties); break; default: @@ -78,7 +68,7 @@ static IonSerializer serializerForIon(final IonType ionType, final SerDeProperti /** * Gets an Ion serializer based on an object inspector and serde properties. */ - static IonSerializer serializerForObjectInspector(final ObjectInspector objectInspector, + IonSerializer serializerForObjectInspector(final ObjectInspector objectInspector, final SerDeProperties properties) { return serializerForIon(ionTypeFrom(objectInspector), properties); } diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java similarity index 66% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java index 0eb2fe8..9fa2835 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SequenceSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; @@ -26,10 +14,16 @@ * Serializer for sequences. */ class SequenceSerializer implements IonSerializer { + private final IonSerializerFactory ionSerializerFactory; private final SerDeProperties properties; private final IonType sequenceType; - SequenceSerializer(final SerDeProperties properties, final IonType sequenceType) { + SequenceSerializer( + final IonSerializerFactory ionSerializerFactory, + final SerDeProperties properties, + final IonType sequenceType + ) { + this.ionSerializerFactory = ionSerializerFactory; this.properties = properties; this.sequenceType = sequenceType; } @@ -44,7 +38,7 @@ public void serialize(final IonWriter writer, for (int i = 0; i < listObjectInspector.getListLength(data); i++) { final ObjectInspector elementObjectInspector = listObjectInspector.getListElementObjectInspector(); - final IonSerializer elementSerializer = IonSerializerFactory.serializerForObjectInspector( + final IonSerializer elementSerializer = ionSerializerFactory.serializerForObjectInspector( elementObjectInspector, properties); diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java new file mode 100644 index 0000000..14b562d --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +/** + * String serializer. + */ +class StringSerializer extends AbstractTextSerializer { + + @Override + protected void writeText(final IonWriter writer, final String text) throws IOException { + writer.writeString(text); + } + + @Override + public IonType getIonType() { + return IonType.STRING; + } +} diff --git a/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java new file mode 100644 index 0000000..9a3517a --- /dev/null +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +/** + * Symbol serializer. + */ +class SymbolSerializer extends AbstractTextSerializer { + @Override + public IonType getIonType() { + return IonType.SYMBOL; + } + + @Override + protected void writeText(final IonWriter writer, final String text) throws IOException { + writer.writeSymbol(text); + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java similarity index 75% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java rename to hive-common/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java index 16d96ad..d6c549d 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java +++ b/hive-common/src/main/java/com/amazon/ionhiveserde/serializers/TableSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; @@ -40,15 +28,18 @@ public class TableSerializer extends AbstractStructSerializer { * @param columnNames table column names * @param properties SerDe properties */ - public TableSerializer(final List columnNames, final SerDeProperties properties) { - super(properties); + public TableSerializer( + final IonSerializerFactory ionSerializerFactory, + final List columnNames, final SerDeProperties properties + ) { + super(ionSerializerFactory, properties); this.columnNames = columnNames; serializerByColumn = new HashMap<>(); for (int i = 0; i < columnNames.size(); i++) { final String columnName = columnNames.get(i); final IonType ionType = properties.serializationIonTypeFor(i); - serializerByColumn.put(columnName, IonSerializerFactory.serializerForIon(ionType, properties)); + serializerByColumn.put(columnName, ionSerializerFactory.serializerForIon(ionType, properties)); } } diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt similarity index 52% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt index 24408f0..e2d79ea 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CaseInsensitiveDecoratorTest.kt @@ -1,104 +1,75 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - package com.amazon.ionhiveserde.configuration -import com.amazon.ion.* +import com.amazon.ion.IonValue import com.amazon.ionhiveserde.ION import com.amazon.ionhiveserde.assertMultiEquals import com.amazon.ionhiveserde.assertSequenceWrapper import com.amazon.ionhiveserde.assertStructWrapper +import com.amazon.ionhiveserde.case_insensitive import com.amazon.ionhiveserde.caseinsensitivedecorator.IonSequenceCaseInsensitiveDecorator import com.amazon.ionhiveserde.caseinsensitivedecorator.IonStructCaseInsensitiveDecorator +import com.amazon.ionhiveserde.sequence_for +import com.amazon.ionhiveserde.struct_for import org.junit.Test import kotlin.test.assertEquals class CaseInsensitiveDecoratorTest { - private fun datagram_for(s: String): IonDatagram { - return ION.loader.load(s) - } - - private fun struct_for(s: String): IonStruct { - val v = datagram_for(s).iterator().next() - if (v !is IonStruct) throw IllegalArgumentException("Required an IonStruct, found ${v.javaClass.simpleName}") - - return IonStructCaseInsensitiveDecorator(v) - } - - private fun sequence_for(s: String): IonSequence { - val v = datagram_for(s).iterator().next() - if (v !is IonSequence) throw IllegalArgumentException("Required an IonSequence, found ${v.javaClass.simpleName}") - - return IonSequenceCaseInsensitiveDecorator(v) - } - @Test fun ionStructCaseInsensitiveDecoratorContainsKey() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertEquals(struct.containsKey("foo"), true) assertEquals(struct.containsKey("bar"), false) } @Test fun ionStructCaseInsensitiveDecoratorGetExist() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertEquals(struct.containsKey("Foo"), true) } @Test fun ionStructCaseInsensitiveDecoratorGetNotExist() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertEquals(struct.containsKey("bar"), false) } @Test fun ionStructCaseInsensitiveDecoratorGetIgnoreCase() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertEquals(struct.containsKey("foO"), true) } @Test fun ionStructCaseInsensitiveDecoratorGetRepeatedFieldFound() { - val struct = struct_for(" { Foo: 'Bar', foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'Bar', foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertEquals(struct.get("Foo"), ION.newSymbol("Bar")) assertEquals(struct.get("foo"), ION.newSymbol("bar")) } @Test fun ionStructCaseInsensitiveDecoratorGetRepeatedFieldFoundIgnoringCase() { - val struct = struct_for(" { Foo: 'Bar', foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'Bar', foo: 'bar'}")) as IonStructCaseInsensitiveDecorator assertMultiEquals( - arrayOf(ION.newSymbol("Bar") as IonValue, ION.newSymbol("bar") as IonValue), - struct.get("FOO")) + arrayOf(ION.newSymbol("Bar") as IonValue, ION.newSymbol("bar") as IonValue), + struct.get("FOO") + ) } @Test fun ionStructCaseInsensitiveDecoratorGetStruct() { - val struct = struct_for(" { Foo: {} }") + val struct = case_insensitive(struct_for("{Foo: {}}")) as IonStructCaseInsensitiveDecorator assertStructWrapper(struct.get("Foo")) } @Test fun ionStructCaseInsensitiveDecoratorGetSequence() { - val struct = struct_for(" { Foo: [] }") + val struct = case_insensitive(struct_for("{ Foo: []}")) as IonStructCaseInsensitiveDecorator assertSequenceWrapper(struct.get("Foo")) } @Test fun ionStructCaseInsensitiveDecoratorRemove() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator val s = struct.remove("Foo") assertEquals(0, struct.size()) @@ -107,21 +78,21 @@ class CaseInsensitiveDecoratorTest { @Test fun ionStructCaseInsensitiveDecoratorRemoveStruct() { - val struct = struct_for(" { Foo: {} }") + val struct = case_insensitive(struct_for("{Foo: {}}")) as IonStructCaseInsensitiveDecorator val s = struct.remove("Foo") assertStructWrapper(s) } @Test fun ionStructCaseInsensitiveDecoratorRemoveSequence() { - val struct = struct_for(" { Foo: [] }") + val struct = case_insensitive(struct_for("{Foo: []}")) as IonStructCaseInsensitiveDecorator val s = struct.remove("Foo") assertSequenceWrapper(s) } @Test fun ionStructCaseInsensitiveDecoratorCloneAndRemove() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator val s = struct.cloneAndRemove("Foo") assertEquals(s.size(), 0) @@ -130,7 +101,7 @@ class CaseInsensitiveDecoratorTest { @Test fun ionStructCaseInsensitiveDecoratorCloneAndRetain() { - val struct = struct_for(" { Foo: 'bar' }") + val struct = case_insensitive(struct_for("{Foo: 'bar'}")) as IonStructCaseInsensitiveDecorator val s = struct.cloneAndRetain("Foo") assertEquals(s.size(), 1) @@ -138,27 +109,46 @@ class CaseInsensitiveDecoratorTest { assertStructWrapper(s) } + @Test + fun ionStructCaseInsensitiveDecoratorIteratorNext() { + val struct = case_insensitive(struct_for("{Foo: 1, Bar: '2'}")) as IonStructCaseInsensitiveDecorator + val iter = struct.iterator() + val foo = iter.next() + assertEquals(foo, ION.newInt(1)) + val bar = iter.next() + assertEquals(bar, ION.newSymbol("2")) + } + + @Test + fun ionStructCaseInsensitiveDecoratorIteratorNull() { + val struct = case_insensitive(struct_for("{Foo: null}")) as IonStructCaseInsensitiveDecorator + val iter = struct.iterator() + val v = iter.next() + // Returns Ion null like what Ion container does + assertEquals(v, ION.newNull()) + } + @Test fun ionSequenceCaseInsensitiveDecoratorGet() { - val sequence = sequence_for("[1, '2']") + val sequence = case_insensitive(sequence_for("[1, '2']")) as IonSequenceCaseInsensitiveDecorator assertEquals(sequence[1], ION.newSymbol("2")) } @Test fun ionSequenceCaseInsensitiveDecoratorGetStruct() { - val sequence = sequence_for("[{}]") + val sequence = case_insensitive(sequence_for("[{}]")) as IonSequenceCaseInsensitiveDecorator assertStructWrapper(sequence[0]) } @Test fun ionSequenceCaseInsensitiveDecoratorGetSequence() { - val sequence = sequence_for("[[]]") + val sequence = case_insensitive(sequence_for("[[]]")) as IonSequenceCaseInsensitiveDecorator assertSequenceWrapper(sequence[0]) } @Test fun ionSequenceCaseInsensitiveDecoratorSet() { - val sequence = sequence_for("[1]") + val sequence = case_insensitive(sequence_for("[1]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.set(0, ION.newInt(2)) assertEquals(sequence[0], ION.newInt(2)) @@ -167,21 +157,21 @@ class CaseInsensitiveDecoratorTest { @Test fun ionSequenceCaseInsensitiveDecoratorSetStruct() { - val sequence = sequence_for("[{}]") + val sequence = case_insensitive(sequence_for("[{}]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.set(0, ION.newInt(2)) assertStructWrapper(l) } @Test fun ionSequenceCaseInsensitiveDecoratorSetSequence() { - val sequence = sequence_for("[[]]") + val sequence = case_insensitive(sequence_for("[[]]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.set(0, ION.newInt(2)) assertSequenceWrapper(l) } @Test fun ionSequenceCaseInsensitiveDecoratorRemove() { - val sequence = sequence_for("[1]") + val sequence = case_insensitive(sequence_for("[1]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.removeAt(0) assertEquals(sequence.size, 0) @@ -190,7 +180,7 @@ class CaseInsensitiveDecoratorTest { @Test fun ionSequenceCaseInsensitiveDecoratorRemoveStruct() { - val sequence = sequence_for("[{}]") + val sequence = case_insensitive(sequence_for("[{}]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.removeAt(0) assertStructWrapper(l) @@ -198,7 +188,7 @@ class CaseInsensitiveDecoratorTest { @Test fun ionSequenceCaseInsensitiveDecoratorRemoveSequence() { - val sequence = sequence_for("[[]]") + val sequence = case_insensitive(sequence_for("[[]]")) as IonSequenceCaseInsensitiveDecorator val l = sequence.removeAt(0) assertSequenceWrapper(l) @@ -206,7 +196,7 @@ class CaseInsensitiveDecoratorTest { @Test fun ionSequenceCaseInsensitiveDecoratorListIterator() { - val sequence = sequence_for("[1, [], {}]") + val sequence = case_insensitive(sequence_for("[1, [], {}]")) as IonSequenceCaseInsensitiveDecorator val iter = sequence.listIterator() assertEquals(iter.next(), ION.newInt(1)) val i = iter.next() @@ -217,11 +207,28 @@ class CaseInsensitiveDecoratorTest { @Test fun ionSequenceCaseInsensitiveDecoratorSublist() { - val sequence = sequence_for("[1, [], {}]") + val sequence = case_insensitive(sequence_for("[1, [], {}]")) as IonSequenceCaseInsensitiveDecorator val sublist = sequence.subList(0, 3) // sublist: [1, [], {}] assertEquals(sublist[0], ION.newInt(1)) assertSequenceWrapper(sublist[1]) assertStructWrapper(sublist[2]) } + + @Test + fun ionSequenceCaseInsensitiveDecoratorIteratorNext() { + val sequence = case_insensitive(sequence_for("[1, '2']")) as IonSequenceCaseInsensitiveDecorator + val iter = sequence.iterator() + assertEquals(iter.next(), ION.newInt(1)) + assertEquals(iter.next(), ION.newSymbol("2")) + } + + @Test + fun ionSequenceCaseInsensitiveDecoratorIteratorNull() { + val sequence = case_insensitive(sequence_for("[null]")) as IonSequenceCaseInsensitiveDecorator + val iter = sequence.iterator() + val v = iter.next() + // Returns Ion null like what Ion container does + assertEquals(v, ION.newNull()); + } } diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt similarity index 83% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt index 1a32d67..06d25c2 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/CatalogConfigTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.configuration @@ -115,4 +103,4 @@ class CatalogConfigTest { assertEquals(1, subject.symbolTableImports.size) assertSst(subject.symbolTableImports[0]) } -} \ No newline at end of file +} diff --git a/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt new file mode 100644 index 0000000..f47cbd5 --- /dev/null +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt @@ -0,0 +1,37 @@ +package com.amazon.ionhiveserde.configuration + +import junitparams.JUnitParamsRunner +import junitparams.Parameters +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals + +@RunWith(JUnitParamsRunner::class) +class EncodingConfigTest { + private fun encodings() = IonEncoding.values() + + private fun makeConfig(encoding: String) = EncodingConfig( + MapBasedRawConfiguration(mapOf("ion.encoding" to encoding)) + ) + + @Test + @Parameters(method = "encodings") + fun encoding(encoding: IonEncoding) { + val subject = makeConfig(encoding.name) + + assertEquals(encoding, subject.encoding) + } + + @Test + fun defaultEncoding() { + val subject = EncodingConfig(MapBasedRawConfiguration(mapOf())) + + assertEquals(IonEncoding.BINARY, subject.encoding) + } + + @Test(expected = IllegalArgumentException::class) + fun invalidEncoding() { + makeConfig("not an encoding") + } + +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt similarity index 65% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt index 709b9cb..7aa3db7 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/FailOnOverflowConfigTest.kt @@ -1,18 +1,3 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - package com.amazon.ionhiveserde.configuration import org.junit.Test @@ -21,7 +6,7 @@ import kotlin.test.assertTrue class FailOnOverflowConfigTest { private fun makeConfig(map: Map, columnNames: List) = - com.amazon.ionhiveserde.configuration.FailOnOverflowConfig(MapBasedRawConfiguration(map), columnNames) + FailOnOverflowConfig(MapBasedRawConfiguration(map), columnNames) @Test fun failOnOverflow() { @@ -53,4 +38,4 @@ class FailOnOverflowConfigTest { assertFalse(subject.failOnOverflowFor("column_1")) assertFalse(subject.failOnOverflowFor("not_a_column")) // not a column } -} \ No newline at end of file +} diff --git a/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt new file mode 100644 index 0000000..1016bfd --- /dev/null +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt @@ -0,0 +1,39 @@ +package com.amazon.ionhiveserde.configuration + +import org.junit.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class IgnoreMalformedConfigTest { + private fun makeConfig(encoding: String) = IgnoreMalformedConfig( + MapBasedRawConfiguration(mapOf("ion.ignore_malformed" to encoding)) + ) + + @Test + fun ignoreMalformedTrue() { + val subject = makeConfig("true") + + assertTrue(subject.ignoreMalformed) + } + + @Test + fun ignoreMalformedFalse() { + val subject = makeConfig("false") + + assertFalse(subject.ignoreMalformed) + } + + @Test + fun ignoreMalformedDefault() { + val subject = IgnoreMalformedConfig(MapBasedRawConfiguration(mapOf())) + + assertFalse(subject.ignoreMalformed) + } + + @Test + fun ignoreMalformedNotBoolean() { + val subject = makeConfig("not a boolean") + + assertFalse(subject.ignoreMalformed) + } +} diff --git a/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt new file mode 100644 index 0000000..12ce805 --- /dev/null +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt @@ -0,0 +1,17 @@ +package com.amazon.ionhiveserde.configuration + +import java.util.* + +class MapBasedRawConfiguration(private val map: Map) : com.amazon.ionhiveserde.configuration.source.RawConfiguration { + override fun get(key: String?): Optional { + return Optional.ofNullable(map[key]) + } + + override fun containsKey(propertyKey: String?): Boolean { + return map.containsKey(propertyKey) + } + + override fun getOrDefault(key: String?, defaultValue: String?): String { + return map[key] ?: defaultValue!! + } +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt similarity index 75% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt index eb34884..095c62a 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/PathExtractionConfigTest.kt @@ -1,18 +1,3 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - package com.amazon.ionhiveserde.configuration import com.amazon.ion.IonInt @@ -37,9 +22,9 @@ class PathExtractionConfigTest { "ion.c1.path_extractor" to "(f1)", "ion.c2.path_extractor" to "(obj f2)") - val pathExtractor = com.amazon.ionhiveserde.configuration.PathExtractionConfig( - MapBasedRawConfiguration(configMap), - listOf("c1", "c2") + val pathExtractor = PathExtractionConfig( + MapBasedRawConfiguration(configMap), + listOf("c1", "c2") ).pathExtractor() assertTrue(struct.isEmpty) @@ -64,7 +49,7 @@ class PathExtractionConfigTest { ) val pathExtractor = PathExtractionConfig( - MapBasedRawConfiguration(configMap), + MapBasedRawConfiguration(configMap), listOf("c1") ).pathExtractor() @@ -89,7 +74,7 @@ class PathExtractionConfigTest { ) val pathExtractor = PathExtractionConfig( - MapBasedRawConfiguration(configMap), + MapBasedRawConfiguration(configMap), listOf("c1") ).pathExtractor() diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt similarity index 88% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt index fd8dd49..decd486 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeAsConfigTest.kt @@ -1,18 +1,3 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - package com.amazon.ionhiveserde.configuration import com.amazon.ion.IonType @@ -27,8 +12,9 @@ import kotlin.test.assertEquals @RunWith(JUnitParamsRunner::class) class SerializeAsConfigTest { - private fun makeConfig(map: Map, types: List) = com.amazon.ionhiveserde.configuration.SerializeAsConfig( - MapBasedRawConfiguration(map), types) + private fun makeConfig(map: Map, types: List) = SerializeAsConfig( + MapBasedRawConfiguration(map), types + ) private fun defaultSerializationMappings() = listOf( // fixed @@ -129,4 +115,4 @@ class SerializeAsConfigTest { listOf(TypeInfoFactory.intTypeInfo) ).serializationIonTypeFor(1) } -} \ No newline at end of file +} diff --git a/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt new file mode 100644 index 0000000..eed82e4 --- /dev/null +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt @@ -0,0 +1,36 @@ +package com.amazon.ionhiveserde.configuration + +import junitparams.JUnitParamsRunner +import junitparams.Parameters +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals + +@RunWith(JUnitParamsRunner::class) +class SerializeNullConfigTest { + private fun makeConfig(serializeNull: String) = SerializeNullConfig( + MapBasedRawConfiguration(mapOf("ion.serialize_null" to serializeNull)) + ) + + private fun serializeNullOptions() = SerializeNullStrategy.values() + + @Test + @Parameters(method = "serializeNullOptions") + fun serializeNull(serializeNullStrategy: SerializeNullStrategy) { + val subject = makeConfig(serializeNullStrategy.name) + + assertEquals(serializeNullStrategy, subject.serializeNull) + } + + @Test + fun defaultSerializeNull() { + val subject = SerializeNullConfig(MapBasedRawConfiguration(mapOf())) + + assertEquals(SerializeNullStrategy.OMIT, subject.serializeNull) + } + + @Test(expected = IllegalArgumentException::class) + fun invalidSerializeNull() { + makeConfig("invalid option") + } +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt similarity index 66% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt index 9e1a986..4073044 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/configuration/TimestampOffsetConfigTest.kt @@ -1,18 +1,3 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - package com.amazon.ionhiveserde.configuration import junitparams.JUnitParamsRunner @@ -24,8 +9,9 @@ import kotlin.test.assertEquals @RunWith(JUnitParamsRunner::class) class TimestampOffsetConfigTest { - private fun makeConfig(offset: String) = com.amazon.ionhiveserde.configuration.TimestampOffsetConfig( - MapBasedRawConfiguration(mapOf("ion.timestamp.serialization_offset" to offset))) + private fun makeConfig(offset: String) = TimestampOffsetConfig( + MapBasedRawConfiguration(mapOf("ion.timestamp.serialization_offset" to offset)) + ) private fun parseOffsetValidTestCases(): List> = listOf( listOf("Z", 0), @@ -60,7 +46,7 @@ class TimestampOffsetConfigTest { @Test fun defaultTimestampOffsetInMinutes() { - val subject = com.amazon.ionhiveserde.configuration.TimestampOffsetConfig(MapBasedRawConfiguration(mapOf())) + val subject = TimestampOffsetConfig(MapBasedRawConfiguration(mapOf())) assertEquals(0, subject.timestampOffsetInMinutes) } @@ -84,4 +70,4 @@ class TimestampOffsetConfigTest { fun invalidTimestampOffsetInMinutes(offsetText: String) { makeConfig(offsetText) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt similarity index 59% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt index 659cd69..2b374dc 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonBooleanToBooleanObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -38,4 +26,4 @@ class IonBooleanToBooleanObjectInspectorTest fun get(testCase: ValidTestCase) { assertEquals(testCase.expectedPrimitive, subject.get(testCase.ionValue)) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt similarity index 60% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt index c7aa9e4..898ce84 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToDoubleObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -37,4 +25,4 @@ class IonFloatToDoubleObjectInspectorTest fun get(testCase: ValidTestCase) { assertEquals(testCase.expectedPrimitive, subject.getPrimitiveJavaObject(testCase.ionValue)) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt similarity index 74% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt index e007386..394ec14 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonFloatToFloatObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt similarity index 78% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt index b001054..daa8ea6 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToBigIntObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt similarity index 73% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt index 47c83e4..146187c 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToIntObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -55,4 +43,4 @@ class IonIntToIntObjectInspectorTest val actual = subjectOverflow.get(testCase.ionValue) assertEquals(testCase.expectedPrimitive, actual) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt similarity index 78% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt index bf410b7..0d2f27f 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToSmallIntObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -62,4 +50,4 @@ class IonIntToSmallIntObjectInspectorTest val actual = subjectOverflow.get(testCase.ionValue) assertEquals(testCase.expectedPrimitive, actual) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt similarity index 78% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt index 98b9998..9d42fba 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonIntToTinyIntObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -63,4 +51,4 @@ class IonIntToTinyIntObjectInspectorTest val actual = subjectOverflow.get(testCase.ionValue) assertEquals(testCase.expectedPrimitive, actual) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt similarity index 65% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt index c36b2b7..2c29e3c 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonLobToBinaryObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -28,7 +16,7 @@ import kotlin.test.assertTrue class IonLobToBinaryObjectInspectorTest : AbstractIonPrimitiveJavaObjectInspectorTest() { - // CLOBs are encoded as 7 bit ASCII, see http://amzn.github.io/ion-docs/docs/spec.html#clob + // CLOBs are encoded as 7 bit ASCII, see http://amazon-ion.github.io/ion-docs/docs/spec.html#clob private val raw = "12345678".toByteArray(charset = Charsets.US_ASCII) override val subject = com.amazon.ionhiveserde.objectinspectors.IonLobToBinaryObjectInspector() @@ -48,4 +36,4 @@ class IonLobToBinaryObjectInspectorTest private fun byteArrayEquals(lhs: ByteArray, rhs: ByteArray): Boolean { return lhs.size == rhs.size && lhs.foldIndexed(true) { index, acc, b -> acc && rhs[index] == b } } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt similarity index 61% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt index 5b7f537..2aaffe3 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonNumberToDecimalObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt similarity index 76% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt index 00edb25..cbe04e5 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonSequenceToListObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -43,7 +31,7 @@ class IonSequenceToListObjectInspectorTest { arrayOf(ION.newEmptyList(), ION.newEmptySexp()).map { it.add(ION.newInt(1)) it.add(ION.newInt(2)) - it.add(ION.newInt(3)) + it.add(ION.newNullInt()) it }.toTypedArray() @@ -52,7 +40,7 @@ class IonSequenceToListObjectInspectorTest { fun getListElement(sequence: IonSequence) { assertEquals(ION.newInt(1), subject.getListElement(sequence, 0)) assertEquals(ION.newInt(2), subject.getListElement(sequence, 1)) - assertEquals(ION.newInt(3), subject.getListElement(sequence, 2)) + assertEquals(null, subject.getListElement(sequence, 2)) assertNull(subject.getListElement(sequence, 4)) assertNull(subject.getListElement(sequence, -1)) @@ -79,7 +67,11 @@ class IonSequenceToListObjectInspectorTest { @Parameters(named = "sequences") fun getList(sequence: IonSequence) { val actual = subject.getList(sequence) - assertEquals(sequence, actual) + // We should make sure the null value is inside the list returned by getList() + assertEquals(3, actual.size) + assertEquals(ION.newInt(1), actual[0]) + assertEquals(ION.newInt(2), actual[1]) + assertEquals(null, actual[2]) } @Test @@ -96,4 +88,4 @@ class IonSequenceToListObjectInspectorTest { fun getTypeName() { assertEquals("array", subject.typeName) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt similarity index 65% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt index 31a1400..b5db73e 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToMapObjectInspectorTest.kt @@ -1,23 +1,12 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors -import com.amazon.ion.IonStruct import com.amazon.ionhiveserde.ION +import com.amazon.ionhiveserde.case_insensitive import com.amazon.ionhiveserde.ionNull +import com.amazon.ionhiveserde.struct_for import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory import org.junit.Test @@ -40,11 +29,11 @@ class IonStructToMapObjectInspectorTest { @Test fun getMapValueElement() { - val struct = makeStruct() + val struct = struct_for("{a: 1, b: 2, c: null}") assertEquals(ION.newInt(1), subject.getMapValueElement(struct, ION.newSymbol("a"))) assertEquals(ION.newInt(2), subject.getMapValueElement(struct, ION.newSymbol("b"))) - assertEquals(ION.newInt(3), subject.getMapValueElement(struct, ION.newSymbol("c"))) + assertEquals(null, subject.getMapValueElement(struct, ION.newSymbol("c"))) assertNull(subject.getMapValueElement(struct, ION.newSymbol("d"))) } @@ -56,29 +45,48 @@ class IonStructToMapObjectInspectorTest { @Test(expected = IllegalArgumentException::class) fun getMapValueElementForNullKey() { - assertNull(subject.getMapValueElement(makeStruct(), null)) + assertNull(subject.getMapValueElement(struct_for("{a: 1, b: 2, c: 3}"), null)) } @Test - fun getMap() { - val struct = makeStruct() + fun getMapSize() { + assertEquals(3, subject.getMapSize(struct_for("{a: 1, b: 2, c: null}"))) + } + @Test + fun getMap() { + val struct = struct_for("{a: 1, b: 2, c: null}") val actual = subject.getMap(struct) + + // We should make sure the null value is inside the map returned by getMap() assertEquals(3, actual.size) + assertEquals(ION.newInt(1), actual["a"]) assertEquals(ION.newInt(2), actual["b"]) - assertEquals(ION.newInt(3), actual["c"]) + assertEquals(null, actual["c"]) } @Test - fun getMapForNullData() { - assertNull(subject.getMap(null)) - assertNull(subject.getMap(ionNull)) + fun getMapSizeForCaseInsensitiveDecorator() { + assertEquals(2, subject.getMapSize(case_insensitive(struct_for("{a: 1, b: null}")))) } @Test - fun getMapSize() { - assertEquals(3, subject.getMapSize(makeStruct())) + fun getMapForCaseInsensitiveDecorator() { + val struct = case_insensitive(struct_for("{a: 1, b: null}")) + val actual = subject.getMap(struct) + + // We should make sure the null value is inside the map returned by getMap() + assertEquals(2, actual.size) + + assertEquals(ION.newInt(1), actual["a"]) + assertEquals(null, actual["b"]) + } + + @Test + fun getMapForNullData() { + assertNull(subject.getMap(null)) + assertNull(subject.getMap(ionNull)) } @Test @@ -96,15 +104,4 @@ class IonStructToMapObjectInspectorTest { fun getCategory() { assertEquals(MAP, subject.category) } - - private fun makeStruct(): IonStruct { - val struct = ION.newEmptyStruct() - - struct.add("a", ION.newInt(1)) - struct.add("b", ION.newInt(2)) - struct.add("c", ION.newInt(3)) - - return struct - } - -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt similarity index 79% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt index 08ee968..e47708f 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonStructToStructInspectorTest.kt @@ -1,22 +1,12 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors import com.amazon.ion.IonStruct import com.amazon.ionhiveserde.ION +import com.amazon.ionhiveserde.case_insensitive +import com.amazon.ionhiveserde.struct_for import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.STRUCT import org.apache.hadoop.hive.serde2.objectinspector.StructField @@ -73,9 +63,22 @@ class IonStructToStructInspectorTest { @Test fun getStructFieldsDataAsList() { - val list = subject.getStructFieldsDataAsList(makeStruct()) + val list = subject.getStructFieldsDataAsList(struct_for("{cboolean: true, cint: 1, cnull: null}")) + // We should make sure the null value is inside the list returned by getStructFieldsDataAsList() + assertEquals(3, list.size) assertEquals(ION.newBool(true), list[0]) assertEquals(ION.newInt(1), list[1]) + assertEquals(null, list[2]) + } + + @Test + fun getStructFieldsDataAsListForCaseInsensitiveDecorator() { + val list = subject.getStructFieldsDataAsList(case_insensitive(struct_for("{cboolean: true, cint: 1, cnull: null}"))) + // We should make sure the null value is inside the list returned by getStructFieldsDataAsList() + assertEquals(3, list.size) + assertEquals(ION.newBool(true), list[0]) + assertEquals(ION.newInt(1), list[1]) + assertEquals(null, list[2]) } @Test @@ -125,4 +128,4 @@ class IonStructToStructInspectorTest { assertEquals("int", field.fieldObjectInspector.typeName) assertEquals("", field.fieldComment) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt similarity index 65% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt index 4abff8b..5e7071e 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToCharObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -34,4 +22,4 @@ class IonTextToCharObjectInspectorTest OverflowTestCase(ION.newString("1234567"), HiveChar("12345", 5), HiveCharWritable(HiveChar("12345", 5))), OverflowTestCase(ION.newSymbol("1234567"), HiveChar("12345", 5), HiveCharWritable(HiveChar("12345", 5))) ) -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt similarity index 50% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt index 23338c3..b438200 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToStringObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -27,4 +15,4 @@ class IonTextToStringObjectInspectorTest : AbstractIonPrimitiveJavaObjectInspect ValidTestCase(ION.newSymbol("some string"), "some string", Text("some string")) ) -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt similarity index 66% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt index 94f587f..3f76f97 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTextToVarcharObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -35,4 +23,4 @@ class IonTextToVarcharObjectInspectorTest OverflowTestCase(ION.newString("1234567"), HiveVarchar("12345", 5), HiveVarcharWritable(HiveVarchar("12345", 5))), OverflowTestCase(ION.newSymbol("1234567"), HiveVarchar("12345", 5), HiveVarcharWritable(HiveVarchar("12345", 5))) ) -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt similarity index 72% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt index 4b43cf1..2e889c0 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUnionObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -68,4 +56,4 @@ class IonUnionObjectInspectorTest { fun getCategory() { assertEquals(UNION, subject.category) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt similarity index 52% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt index 5479a9c..098711e 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonUtilTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -43,4 +31,4 @@ class IonUtilTest { fun isIonNullForTypedNull() { assertTrue(isIonNull(ION.newNull(IonType.BOOL))) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt similarity index 74% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt index 9ac8153..995bc32 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonValueToStringObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt similarity index 52% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt rename to hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt index 849f000..1929dd7 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt +++ b/hive-common/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidatorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -39,4 +27,4 @@ class TextMaxLengthValidatorTest { fun validateForInvalidSize() { subject.validate("1234567", -1) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/CustomAssert.kt b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/CustomAssert.kt similarity index 63% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/CustomAssert.kt rename to hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/CustomAssert.kt index 2fe200c..b28671c 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/CustomAssert.kt +++ b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/CustomAssert.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde diff --git a/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/Util.kt b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/Util.kt new file mode 100644 index 0000000..2edbe45 --- /dev/null +++ b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/Util.kt @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde + +import com.amazon.ion.IonDatagram +import com.amazon.ion.IonSequence +import com.amazon.ion.IonStruct +import com.amazon.ion.IonValue +import com.amazon.ion.system.IonSystemBuilder +import com.amazon.ionhiveserde.caseinsensitivedecorator.IonCaseInsensitiveDecorator + +val ION = IonSystemBuilder.standard().build() +val ionNull = ION.newNull() + +fun datagram_for(s: String): IonDatagram { + return ION.loader.load(s) +} + +fun struct_for(s: String): IonStruct { + val v = datagram_for(s).iterator().next() + if (v !is IonStruct) throw IllegalArgumentException("Required an IonStruct, found ${v.javaClass.simpleName}") + + return v +} + +fun sequence_for(s: String): IonSequence { + val v = datagram_for(s).iterator().next() + if (v !is IonSequence) throw IllegalArgumentException("Required an IonSequence, found ${v.javaClass.simpleName}") + + return v +} + +fun case_insensitive(v: IonValue): IonValue { + return IonCaseInsensitiveDecorator.wrapValue(v) +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt similarity index 78% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt rename to hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt index 959264f..0788a37 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt +++ b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractIonPrimitiveJavaObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt similarity index 80% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt rename to hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt index 169d54f..d73ef38 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt +++ b/hive-common/src/testFixtures/kotlin/com/amazon/ionhiveserde/objectinspectors/AbstractOverflowablePrimitiveObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -70,4 +58,4 @@ abstract class AbstractOverflowablePrimitiveObjectInspectorTest) { assertEquals(testCase.expectedPrimitive, subjectOverflow.getPrimitiveJavaObject(testCase.ionValue)) } -} \ No newline at end of file +} diff --git a/hive2/build.gradle.kts b/hive2/build.gradle.kts new file mode 100644 index 0000000..bdf5354 --- /dev/null +++ b/hive2/build.gradle.kts @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +plugins { + id("ion-hive-serde.dependencies") + id("ion-hive-serde.conventions") + id("ion-hive-serde.publishing") +} + +configurations { + implementation { extendsFrom(bundled.get(), hive2Runtime.get()) } + testImplementation { extendsFrom(testFramework.get()) } +} + +dependencies { + bundled(project(":hive-common")) + testFramework(testFixtures(project(":hive-common"))) +} diff --git a/hive2/settings.gradle.kts b/hive2/settings.gradle.kts new file mode 100644 index 0000000..c381544 --- /dev/null +++ b/hive2/settings.gradle.kts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include(":hive-common") diff --git a/hive2/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java b/hive2/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java new file mode 100644 index 0000000..29c52fb --- /dev/null +++ b/hive2/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java @@ -0,0 +1,47 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde; + +import com.amazon.ionhiveserde.configuration.SerDeProperties; +import com.amazon.ionhiveserde.objectinspectors.IonTimestampToDateObjectInspector; +import com.amazon.ionhiveserde.objectinspectors.IonTimestampToTimestampObjectInspector; +import com.amazon.ionhiveserde.objectinspectors.factories.IonObjectInspectorFactory; +import com.amazon.ionhiveserde.serializers.Hive2IonSerializerFactory; +import com.amazon.ionhiveserde.serializers.TableSerializer; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +/** + *

+ * Hive SerDe for the Amazon Ion data format. + *

+ *

+ * For more information on Hive SerDes see wiki. + *

+ */ +public class IonHiveSerDe extends AbstractIonHiveSerDe { + + private static final IonObjectInspectorFactory ION_OBJECT_INSPECTOR_FACTORY = + new IonObjectInspectorFactory(new IonTimestampToDateObjectInspector(), + new IonTimestampToTimestampObjectInspector()); + + @Override + protected TableSerializer getTableSerializer(final SerDeProperties properties) { + return new TableSerializer( + new Hive2IonSerializerFactory(), + properties.getColumnNames(), + properties + ); + } + + @Override + protected ObjectInspector getObjectInspectorForTable(final SerDeProperties properties) { + final StructTypeInfo tableInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo( + serDeProperties.getColumnNames(), + serDeProperties.getColumnTypes()); + return ION_OBJECT_INSPECTOR_FACTORY.objectInspectorForTable(tableInfo, serDeProperties); + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java b/hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java similarity index 67% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java rename to hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java index 601993c..af9c502 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java +++ b/hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java b/hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java similarity index 71% rename from serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java rename to hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java index 7501465..ee0bb1c 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java +++ b/hive2/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors; diff --git a/hive2/src/main/java/com/amazon/ionhiveserde/serializers/Hive2IonSerializerFactory.java b/hive2/src/main/java/com/amazon/ionhiveserde/serializers/Hive2IonSerializerFactory.java new file mode 100644 index 0000000..d449470 --- /dev/null +++ b/hive2/src/main/java/com/amazon/ionhiveserde/serializers/Hive2IonSerializerFactory.java @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ionhiveserde.configuration.SerDeProperties; + +public class Hive2IonSerializerFactory extends IonSerializerFactory { + @Override + protected IonSerializer newTimestampSerializer(final SerDeProperties properties) { + return new TimestampSerializer(properties.getTimestampOffsetInMinutes()); + } +} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java b/hive2/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java similarity index 78% rename from serde/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java rename to hive2/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java index 3f20521..ac39efe 100644 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java +++ b/hive2/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.serializers; diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt b/hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt similarity index 75% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt rename to hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt index be39671..ce16eb0 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt +++ b/hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -46,4 +34,4 @@ class IonTimestampToDateObjectInspectorTest // assert on LocalDate because Date keeps the time component internally... assertEquals(testCase.expectedPrimitive.toLocalDate(), actualDate.toLocalDate()) } -} \ No newline at end of file +} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt b/hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt similarity index 71% rename from serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt rename to hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt index 806a3d2..b644f83 100644 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt +++ b/hive2/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.objectinspectors @@ -34,4 +22,4 @@ class IonTimestampToTimestampObjectInspectorTest ValidTestCase(ION.newTimestamp(Timestamp.valueOf("2018-08-15T01:01Z")), java.sql.Timestamp(1534294860000L), TimestampWritable(java.sql.Timestamp(1534294860000L))), ValidTestCase(ION.newTimestamp(Timestamp.valueOf("2018-08-15T01:01:01Z")), java.sql.Timestamp(1534294861000L), TimestampWritable(java.sql.Timestamp(1534294861000L))) ) -} \ No newline at end of file +} diff --git a/hive3/build.gradle.kts b/hive3/build.gradle.kts new file mode 100644 index 0000000..87dea45 --- /dev/null +++ b/hive3/build.gradle.kts @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +plugins { + id("ion-hive-serde.dependencies") + id("ion-hive-serde.conventions") + id("ion-hive-serde.publishing") +} + +configurations { + implementation { extendsFrom(bundled.get(), hive3Runtime.get()) } + testImplementation { extendsFrom(testFramework.get()) } +} + +dependencies { + bundled(project(":hive-common")) + testFramework(testFixtures(project(":hive-common"))) +} diff --git a/hive3/settings.gradle.kts b/hive3/settings.gradle.kts new file mode 100644 index 0000000..c381544 --- /dev/null +++ b/hive3/settings.gradle.kts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include(":hive-common") diff --git a/hive3/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java b/hive3/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java new file mode 100644 index 0000000..a3efc8a --- /dev/null +++ b/hive3/src/main/java/com/amazon/ionhiveserde/IonHiveSerDe.java @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde; + +import com.amazon.ionhiveserde.configuration.SerDeProperties; +import com.amazon.ionhiveserde.objectinspectors.IonTimestampToDateObjectInspector; +import com.amazon.ionhiveserde.objectinspectors.IonTimestampToTimestampObjectInspector; +import com.amazon.ionhiveserde.objectinspectors.factories.IonObjectInspectorFactory; +import com.amazon.ionhiveserde.serializers.Hive3IonSerializerFactory; +import com.amazon.ionhiveserde.serializers.TableSerializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +/** + *

+ * Hive SerDe for the Amazon Ion data format. + *

+ *

+ * For more information on Hive SerDes see wiki. + *

+ */ +public class IonHiveSerDe extends AbstractIonHiveSerDe { + + private static final IonObjectInspectorFactory ION_OBJECT_INSPECTOR_FACTORY = + new IonObjectInspectorFactory(new IonTimestampToDateObjectInspector(), + new IonTimestampToTimestampObjectInspector()); + + @Override + protected TableSerializer getTableSerializer(final SerDeProperties properties) { + return new TableSerializer( + new Hive3IonSerializerFactory(), + properties.getColumnNames(), + properties + ); + } + + @Override + protected ObjectInspector getObjectInspectorForTable(final SerDeProperties properties) { + final StructTypeInfo tableInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo( + serDeProperties.getColumnNames(), + serDeProperties.getColumnTypes()); + return ION_OBJECT_INSPECTOR_FACTORY.objectInspectorForTable(tableInfo, serDeProperties); + } +} diff --git a/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java b/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java new file mode 100644 index 0000000..a6ae7a5 --- /dev/null +++ b/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspector.java @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors; + +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonValue; + +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +/** + * Adapts an {@link IonTimestamp} for the date Hive type. + */ +public class IonTimestampToDateObjectInspector extends AbstractIonPrimitiveJavaObjectInspector implements + DateObjectInspector { + + public IonTimestampToDateObjectInspector() { + super(TypeInfoFactory.dateTypeInfo); + } + + @Override + public DateWritableV2 getPrimitiveWritableObject(final Object o) { + if (IonUtil.isIonNull((IonValue) o)) { + return null; + } + + return new DateWritableV2(getPrimitiveJavaObject((IonTimestamp) o)); + } + + @Override + public Date getPrimitiveJavaObject(final Object o) { + if (IonUtil.isIonNull((IonValue) o)) { + return null; + } + + return getPrimitiveJavaObject((IonTimestamp) o); + } + + private Date getPrimitiveJavaObject(final IonTimestamp ionValue) { + return Date.ofEpochMilli(ionValue.getMillis()); + } +} + diff --git a/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java b/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java new file mode 100644 index 0000000..85fae87 --- /dev/null +++ b/hive3/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspector.java @@ -0,0 +1,48 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors; + +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonValue; +import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +/** + * Adapts an {@link IonTimestamp} for the timestamp Hive type. + */ +public class IonTimestampToTimestampObjectInspector extends AbstractIonPrimitiveJavaObjectInspector implements + TimestampObjectInspector { + + public IonTimestampToTimestampObjectInspector() { + super(TypeInfoFactory.timestampTypeInfo); + } + + @Override + public TimestampWritableV2 getPrimitiveWritableObject(final Object o) { + if (IonUtil.isIonNull((IonValue) o)) { + return null; + } + + return new TimestampWritableV2(getPrimitiveJavaObject((IonTimestamp) o)); + } + + @Override + public Timestamp getPrimitiveJavaObject(final Object o) { + if (IonUtil.isIonNull((IonValue) o)) { + return null; + } + + return getPrimitiveJavaObject((IonTimestamp) o); + } + + private Timestamp getPrimitiveJavaObject(final IonTimestamp ionValue) { + // Hive timestamps don't have offset so we always map the ion timestamp to UTC + // IonTimestamp.getMillis() is milliseconds from 1970-01-01T00:00:00.000Z + + return Timestamp.ofEpochMilli(ionValue.getMillis()); + } +} diff --git a/hive3/src/main/java/com/amazon/ionhiveserde/serializers/Hive3IonSerializerFactory.java b/hive3/src/main/java/com/amazon/ionhiveserde/serializers/Hive3IonSerializerFactory.java new file mode 100644 index 0000000..8fc0309 --- /dev/null +++ b/hive3/src/main/java/com/amazon/ionhiveserde/serializers/Hive3IonSerializerFactory.java @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ionhiveserde.configuration.SerDeProperties; + +public class Hive3IonSerializerFactory extends IonSerializerFactory { + @Override + protected IonSerializer newTimestampSerializer(final SerDeProperties properties) { + return new TimestampSerializer(properties.getTimestampOffsetInMinutes()); + } +} diff --git a/hive3/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java b/hive3/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java new file mode 100644 index 0000000..52b96d1 --- /dev/null +++ b/hive3/src/main/java/com/amazon/ionhiveserde/serializers/TimestampSerializer.java @@ -0,0 +1,63 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.serializers; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.Timestamp; +import java.io.IOException; + +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; + +/** + * Timestamp serializer. + */ +class TimestampSerializer implements IonSerializer { + + private final Integer offsetInMinutes; + + TimestampSerializer(final Integer offsetInMinutes) { + this.offsetInMinutes = offsetInMinutes; + } + + @Override + public void serialize(final IonWriter writer, + final Object data, + final ObjectInspector objectInspector) throws IOException { + + final PrimitiveCategory category = ((PrimitiveObjectInspector) objectInspector).getPrimitiveCategory(); + switch (category) { + case DATE: + final Date date = ((DateObjectInspector) objectInspector) + .getPrimitiveJavaObject(data); + + Timestamp ionTimestamp = Timestamp.forDay(date.getYear(), date.getMonth(), + date.getDay()); + writer.writeTimestamp(ionTimestamp); + break; + + case TIMESTAMP: + final org.apache.hadoop.hive.common.type.Timestamp hiveTimestamp = + ((TimestampObjectInspector) objectInspector).getPrimitiveJavaObject(data); + + final Timestamp timestampWithOffset = Timestamp.forMillis(hiveTimestamp.toEpochMilli(), + offsetInMinutes); + writer.writeTimestamp(timestampWithOffset); + break; + + default: + throw new IllegalArgumentException("Invalid object category for timestamp serializer: " + category); + } + } + + @Override + public IonType getIonType() { + return IonType.TIMESTAMP; + } +} diff --git a/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt b/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt new file mode 100644 index 0000000..4f258f7 --- /dev/null +++ b/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToDateObjectInspectorTest.kt @@ -0,0 +1,43 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors + +import com.amazon.ion.IonTimestamp +import com.amazon.ion.Timestamp +import com.amazon.ionhiveserde.ION +import junitparams.JUnitParamsRunner +import junitparams.Parameters +import org.apache.hadoop.hive.common.type.Date +import org.apache.hadoop.hive.serde2.io.DateWritableV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals + +@RunWith(JUnitParamsRunner::class) +class IonTimestampToDateObjectInspectorTest + : AbstractIonPrimitiveJavaObjectInspectorTest() { + + override val subject = com.amazon.ionhiveserde.objectinspectors.IonTimestampToDateObjectInspector() + override fun validTestCases() = listOf( + validTestCase("2018T", 1514764800000L,1514764800000L), + validTestCase("2018-08T", 1533081600000L,1533081600000L), + validTestCase("2018-08-15T", 1534291200000L,1534291200000L), + validTestCase("2018-08-15T01:01Z", 1534291200000L,1534291200000L), + validTestCase("2018-08-15T01:01:01Z",1534291200000L,1534291200000L) + ) + + private fun validTestCase(ionFormattedTimestamp: String, dateEpochMilli: Long, dateWritableEpochMilli: Long) = + ValidTestCase( + ION.newTimestamp(Timestamp.valueOf(ionFormattedTimestamp)), + Date.ofEpochMilli(dateEpochMilli), + DateWritableV2(Date.ofEpochMilli(dateWritableEpochMilli))) + + @Test + @Parameters(method = "validTestCases") + override fun getPrimitiveJavaObject(testCase: ValidTestCase) { + val actualDate = subject.getPrimitiveJavaObject(testCase.ionValue) as Date + // assert on epoch milli because Date keeps the time component internally... + assertEquals(testCase.expectedPrimitive.toEpochMilli(), actualDate.toEpochMilli()) + } +} diff --git a/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt b/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt new file mode 100644 index 0000000..29ca481 --- /dev/null +++ b/hive3/src/test/kotlin/com/amazon/ionhiveserde/objectinspectors/IonTimestampToTimestampObjectInspectorTest.kt @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ionhiveserde.objectinspectors + +import com.amazon.ion.IonTimestamp +import com.amazon.ionhiveserde.ION +import junitparams.JUnitParamsRunner +import org.apache.hadoop.hive.common.type.Timestamp +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2 +import org.junit.runner.RunWith + +@RunWith(JUnitParamsRunner::class) +class IonTimestampToTimestampObjectInspectorTest + : AbstractIonPrimitiveJavaObjectInspectorTest() { + + override val subject = IonTimestampToTimestampObjectInspector() + override fun validTestCases() = listOf( + validTestCase("2018T" , 1514764800000L, 1514764800000L), + validTestCase("2018-08T" , 1533081600000L, 1533081600000L), + validTestCase("2018-08-15T" , 1534291200000L, 1534291200000L), + validTestCase("2018-08-15T01:01Z" , 1534294860000L, 1534294860000L), + validTestCase("2018-08-15T01:01:01Z", 1534294861000L, 1534294861000L) + ) + + private fun validTestCase(ionFormattedTimestamp: String, timestampEpochMilli: Long, timestampWritableEpochMilli: Long) = + ValidTestCase( + ION.newTimestamp(com.amazon.ion.Timestamp.valueOf(ionFormattedTimestamp)), + Timestamp.ofEpochMilli(timestampEpochMilli), + TimestampWritableV2(Timestamp.ofEpochMilli(timestampWritableEpochMilli))) +} diff --git a/integration-test/build.gradle b/integration-test/build.gradle index 2c2fb03..3af747c 100644 --- a/integration-test/build.gradle +++ b/integration-test/build.gradle @@ -1,17 +1,10 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +plugins { + id("ion-hive-serde.dependencies") + id("ion-hive-serde.conventions") +} task createTmpDir { doLast { @@ -37,11 +30,11 @@ task stopDocker(type: Exec) { } task copySerDeJar { - dependsOn ':serde:singleJar' + dependsOn ':hive2:shadowJar' doLast { copy { - from file("../serde/build/libs/ion-hive-serde-all-${project(':serde').version}.jar") + from file("../serde/build/libs/ion-hive2-serde-${project(':hive2').version}-all.jar") into file("./docker-tmp/serde/") rename '.*\\.jar', 'serde-all.jar' } @@ -72,7 +65,7 @@ task testTearDown { * - stop docker containers * - remove shared directories */ -test { +tasks.withType(Test) { dependsOn testSetUp finalizedBy 'testTearDown' @@ -81,12 +74,15 @@ test { include "**/*Suite.class" } -dependencies { - testCompile project(":serde") +def hive_version = '[2.3.0,2.4.0)' - testCompile "org.apache.hive:hive-jdbc:${hive_version}" +configurations { + testImplementation { + extendsFrom(testFramework, hive2Runtime) + } +} - // compile only dependencies are not transitive so we need to redeclare some here - testCompile "org.apache.hive:hive-exec:${hive_version}", { transitive = false } - testCompile "org.apache.hadoop:hadoop-common:${hadoop_version}" +dependencies { + testImplementation project(":hive2") + testImplementation "org.apache.hive:hive-jdbc:${hive_version}" } diff --git a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/AllTestsSuite.java b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/AllTestsSuite.java index 692ca4e..35e70be 100644 --- a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/AllTestsSuite.java +++ b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/AllTestsSuite.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest; diff --git a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Base.java b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Base.java index f9438c8..07146c0 100644 --- a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Base.java +++ b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Base.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest; diff --git a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Lifecycle.java b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Lifecycle.java index 04741e0..7fed04c 100644 --- a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Lifecycle.java +++ b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/Lifecycle.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest; diff --git a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestLifecycle.java b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestLifecycle.java index 01204e1..0f91f03 100644 --- a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestLifecycle.java +++ b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestLifecycle.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest; diff --git a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestMode.java b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestMode.java index 98efac7..f886434 100644 --- a/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestMode.java +++ b/integration-test/src/test/java/com/amazon/ionhiveserde/integrationtest/TestMode.java @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest; diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/FileSystemUtil.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/FileSystemUtil.kt index 6a31189..173d6f5 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/FileSystemUtil.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/FileSystemUtil.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 /** * Utility functions for dealing with the local file system. diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/IonUtil.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/IonUtil.kt index 5612aca..05ddb56 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/IonUtil.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/IonUtil.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 /** * Ion related utility methods to be used in the test scope. @@ -55,4 +43,4 @@ fun IonStruct.keys(): Sequence = this.asSequence().map { it.fieldName } /** Write all values from a string containing an ion document */ fun IonWriter.writeValues(ionDocument: String) = - IonReaderBuilder.standard().build(ionDocument).use { r -> this.writeValues(r) } \ No newline at end of file + IonReaderBuilder.standard().build(ionDocument).use { r -> this.writeValues(r) } diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/DockerUtil.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/DockerUtil.kt index 979d430..bf23fba 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/DockerUtil.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/DockerUtil.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.docker @@ -78,4 +66,4 @@ fun runInDocker(command: String, timeoutInMs: Long = DEFAULT_SHELL_TIMEOUT_MS): } } -class ShellException(message: String) : Exception(message) \ No newline at end of file +class ShellException(message: String) : Exception(message) diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/HDFS.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/HDFS.kt index b576983..c464f81 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/HDFS.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/HDFS.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.docker @@ -21,7 +9,7 @@ package com.amazon.ionhiveserde.integrationtest.docker * There is something wrong with the hadoop docker config that's blocking interacting with HDFS from outside the docker * container. To work around that we are using shell to send commands to docker to manipulate HDFS. * - * See https://github.com/amzn/ion-hive-serde/issues/37 for more info + * See https://github.com/amazon-ion/ion-hive-serde/issues/37 for more info */ object HDFS { diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/Hive.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/Hive.kt index 3efdbee..6824509 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/Hive.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/docker/Hive.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.docker diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/setup/TestData.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/setup/TestData.kt index 52955c6..f031434 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/setup/TestData.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/setup/TestData.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.setup diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/FailOnOverflowTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/FailOnOverflowTest.kt index 5ac3e58..0146bcd 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/FailOnOverflowTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/FailOnOverflowTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/IgnoreMalformedTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/IgnoreMalformedTest.kt index 3e73d46..e50ca5a 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/IgnoreMalformedTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/IgnoreMalformedTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/NullMappingTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/NullMappingTest.kt index 2bdca8c..ef12433 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/NullMappingTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/NullMappingTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/PathExtractorTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/PathExtractorTest.kt index 1ea17ff..67d45ac 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/PathExtractorTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/PathExtractorTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SerializeAsTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SerializeAsTest.kt index 35d6eec..cb4f97e 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SerializeAsTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SerializeAsTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SharedSymbolTableTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SharedSymbolTableTest.kt index 1631945..a781502 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SharedSymbolTableTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/SharedSymbolTableTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TimestampSerializationOffsetTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TimestampSerializationOffsetTest.kt index 572e8fe..adba906 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TimestampSerializationOffsetTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TimestampSerializationOffsetTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TypeMappingTest.kt b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TypeMappingTest.kt index ca5dfa2..452f97b 100644 --- a/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TypeMappingTest.kt +++ b/integration-test/src/test/kotlin/com/amazon/ionhiveserde/integrationtest/tests/TypeMappingTest.kt @@ -1,17 +1,5 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ionhiveserde.integrationtest.tests diff --git a/serde/build.gradle b/serde/build.gradle deleted file mode 100644 index f525bd1..0000000 --- a/serde/build.gradle +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -version = "0.7.0-SNAPSHOT" -group = "com.amazon.ion" - -apply plugin: 'java-library' -apply plugin: "maven-publish" -apply plugin: "signing" - -jar.baseName = 'ion-hive-serde' - -// only applicable for src/main/java source set -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -// Publishing -// ------------------------------------------------------------------------------- - -task sourcesJar(type: Jar) { - from sourceSets.main.allJava - classifier = 'sources' -} - -task javadocJar(type: Jar) { - from javadoc - classifier = 'javadoc' -} - -task singleJar(dependsOn: 'build', type: Jar) { - baseName = "${jar.baseName}-all" - from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } - with jar -} - - -publishing { - publications { - mavenJava(MavenPublication) { - artifactId = "ion-hive-serde" - - from components.java - artifact sourcesJar - artifact javadocJar - - pom { - name = "Ion Hive SerDe" - packaging = "jar" - url = "https://github.com/amzn/ion-hive-serde" - description = "A Apache Hive SerDe (short for serializer/deserializer) for the Ion file format." - scm { - connection = "scm:git@github.com:amzn/ion-hive-serde.git" - developerConnection = "scm:git@github.com:amzn/ion-hive-serde.git" - url = "git@github.com:amzn/ion-hive-serde.git" - } - - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - - developers { - developer { - name = "Amazon Ion Team" - email = "ion-team@amazon.com" - organization = "Amazon Labs" - organizationUrl = "https://github.com/amzn" - } - } - } - } - } - repositories { - maven { - url "https://aws.oss.sonatype.org/service/local/staging/deploy/maven2" - credentials { - username ossrhUsername - password ossrhPassword - } - } - } -} - -signing { - sign publishing.publications.mavenJava -} - -dependencies { - // Included in the single-jar - compile "com.amazon.ion:ion-java:${ionjava_version}" - compile "com.amazon.ion:ion-java-path-extraction:${pathextraction_version}" - - // Provided by the Hive installation - compileOnly "org.apache.hive:hive-serde:${hive_version}" - compileOnly "org.apache.hive:hive-exec:${hive_version}", { transitive = false } - compileOnly "org.apache.hadoop:hadoop-common:${hadoop_version}" - compileOnly "org.apache.hadoop:hadoop-main:${hadoop_version}" - compileOnly "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}" -} - diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java b/serde/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java deleted file mode 100644 index f2efe99..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/HadoopProperties.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration; - -import com.amazon.ionhiveserde.configuration.source.RawConfiguration; - -/** - * Encapsulates Ion related Hadoop job properties. - */ -public class HadoopProperties extends BaseProperties { - - /** - * Constructor. - * - * @param configuration raw configuration. - */ - public HadoopProperties(final RawConfiguration configuration) { - super(configuration); - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java b/serde/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java deleted file mode 100644 index c5a03df..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/IonEncoding.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration; - -/** - * Possible Ion encodings. - */ -public enum IonEncoding { - /** - * Ion binary. - */ - BINARY, - - /** - * Ion text. - */ - TEXT -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java b/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java deleted file mode 100644 index 54e674f..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/configuration/SerializeNullStrategy.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration; - -/** - * Possible strategies for the serialize_null property. - */ -public enum SerializeNullStrategy { - /** - * Omit nulls. - */ - OMIT, - - /** - * Serialize strongly typed nulls. - */ - TYPED, - - /** - * Serialize untyped nulls. - */ - UNTYPED; -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java b/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java deleted file mode 100644 index 0f65a3d..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/IonUtil.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.objectinspectors; - -import com.amazon.ion.IonValue; - -/** - * Utility methods for Ion values. - */ -public class IonUtil { - - static boolean isIonNull(final IonValue ionValue) { - return ionValue == null || ionValue.isNullValue(); - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java b/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java deleted file mode 100644 index b805a3c..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/TextMaxLengthValidator.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.objectinspectors; - -/** - * Validates a {@link String} length, throwing if the string is longer than expected. - */ -class TextMaxLengthValidator { - - /** - * Validates text size. - * - * @throws IllegalArgumentException if the string is longer than max length - */ - void validate(final String text, final int maxLength) { - if (text.length() > maxLength) { - throw new IllegalArgumentException( - "text is longer than allowed. Size: " + text.length() + ", max: " + maxLength); - } - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java b/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java deleted file mode 100644 index d3f9c24..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/objectinspectors/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -/* - * ObjectInspectors are adapters from Ion types into Hive types. This package contains one ObjectInspector per - * valid Ion to Hive type combination - */ -package com.amazon.ionhiveserde.objectinspectors; - diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java b/serde/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java deleted file mode 100644 index d2371d3..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/BlobSerializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.serializers; - -import com.amazon.ion.IonType; -import com.amazon.ion.IonWriter; -import java.io.IOException; - -/** - * Serializer for Ion blob. - */ -class BlobSerializer extends AbstractLobSerializer { - - @Override - protected void writeValue(final IonWriter writer, final byte[] bytes) throws IOException { - writer.writeBlob(bytes); - } - - @Override - public IonType getIonType() { - return IonType.BLOB; - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java b/serde/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java deleted file mode 100644 index fc3ea03..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/ClobSerializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.serializers; - -import com.amazon.ion.IonType; -import com.amazon.ion.IonWriter; -import java.io.IOException; - -/** - * Serializer for clob. - */ -class ClobSerializer extends AbstractLobSerializer { - - @Override - public IonType getIonType() { - return IonType.CLOB; - } - - @Override - protected void writeValue(final IonWriter writer, final byte[] bytes) throws IOException { - writer.writeClob(bytes); - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java b/serde/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java deleted file mode 100644 index faa0de8..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/ColumnStructSerializer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.serializers; - -import com.amazon.ionhiveserde.configuration.SerDeProperties; - -/** - * Struct serializer. - */ -class ColumnStructSerializer extends AbstractStructSerializer { - - ColumnStructSerializer(final SerDeProperties properties) { - super(properties); - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java b/serde/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java deleted file mode 100644 index 8bb5753..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/StringSerializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.serializers; - -import com.amazon.ion.IonType; -import com.amazon.ion.IonWriter; -import java.io.IOException; - -/** - * String serializer. - */ -class StringSerializer extends AbstractTextSerializer { - - @Override - protected void writeText(final IonWriter writer, final String text) throws IOException { - writer.writeString(text); - } - - @Override - public IonType getIonType() { - return IonType.STRING; - } -} diff --git a/serde/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java b/serde/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java deleted file mode 100644 index 79cdd04..0000000 --- a/serde/src/main/java/com/amazon/ionhiveserde/serializers/SymbolSerializer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.serializers; - -import com.amazon.ion.IonType; -import com.amazon.ion.IonWriter; -import java.io.IOException; - -/** - * Symbol serializer. - */ -class SymbolSerializer extends AbstractTextSerializer { - @Override - public IonType getIonType() { - return IonType.SYMBOL; - } - - @Override - protected void writeText(final IonWriter writer, final String text) throws IOException { - writer.writeSymbol(text); - } -} diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/Util.kt b/serde/src/test/kotlin/com/amazon/ionhiveserde/Util.kt deleted file mode 100644 index fddcbcc..0000000 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/Util.kt +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde - -import com.amazon.ion.system.IonSystemBuilder - -internal val ION = IonSystemBuilder.standard().build() -internal val ionNull = ION.newNull() diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt b/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt deleted file mode 100644 index 6363d25..0000000 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/EncodingConfigTest.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration - -import junitparams.JUnitParamsRunner -import junitparams.Parameters -import org.junit.Test -import org.junit.runner.RunWith -import kotlin.test.assertEquals - -@RunWith(JUnitParamsRunner::class) -class EncodingConfigTest { - private fun encodings() = com.amazon.ionhiveserde.configuration.IonEncoding.values() - - private fun makeConfig(encoding: String) = com.amazon.ionhiveserde.configuration.EncodingConfig( - MapBasedRawConfiguration(mapOf("ion.encoding" to encoding))) - - @Test - @Parameters(method = "encodings") - fun encoding(encoding: com.amazon.ionhiveserde.configuration.IonEncoding) { - val subject = makeConfig(encoding.name) - - assertEquals(encoding, subject.encoding) - } - - @Test - fun defaultEncoding() { - val subject = com.amazon.ionhiveserde.configuration.EncodingConfig(MapBasedRawConfiguration(mapOf())) - - assertEquals(com.amazon.ionhiveserde.configuration.IonEncoding.BINARY, subject.encoding) - } - - @Test(expected = IllegalArgumentException::class) - fun invalidEncoding() { - makeConfig("not an encoding") - } - -} \ No newline at end of file diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt b/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt deleted file mode 100644 index 66cf3c7..0000000 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/IgnoreMalformedConfigTest.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration - -import org.junit.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class IgnoreMalformedConfigTest { - private fun makeConfig(encoding: String) = com.amazon.ionhiveserde.configuration.IgnoreMalformedConfig( - MapBasedRawConfiguration(mapOf("ion.ignore_malformed" to encoding))) - - @Test - fun ignoreMalformedTrue() { - val subject = makeConfig("true") - - assertTrue(subject.ignoreMalformed) - } - - @Test - fun ignoreMalformedFalse() { - val subject = makeConfig("false") - - assertFalse(subject.ignoreMalformed) - } - - @Test - fun ignoreMalformedDefault() { - val subject = com.amazon.ionhiveserde.configuration.IgnoreMalformedConfig(MapBasedRawConfiguration(mapOf())) - - assertFalse(subject.ignoreMalformed) - } - - @Test - fun ignoreMalformedNotBoolean() { - val subject = makeConfig("not a boolean") - - assertFalse(subject.ignoreMalformed) - } -} \ No newline at end of file diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt b/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt deleted file mode 100644 index 5b4f08f..0000000 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/MapBasedRawConfiguration.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration - -import java.util.* - -class MapBasedRawConfiguration(private val map: Map) : com.amazon.ionhiveserde.configuration.source.RawConfiguration { - override fun get(key: String?): Optional { - return Optional.ofNullable(map[key]) - } - - override fun containsKey(propertyKey: String?): Boolean { - return map.containsKey(propertyKey) - } - - override fun getOrDefault(key: String?, defaultValue: String?): String { - return map[key] ?: defaultValue!! - } -} \ No newline at end of file diff --git a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt b/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt deleted file mode 100644 index 9981f36..0000000 --- a/serde/src/test/kotlin/com/amazon/ionhiveserde/configuration/SerializeNullConfigTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ionhiveserde.configuration - -import junitparams.JUnitParamsRunner -import junitparams.Parameters -import org.junit.Test -import org.junit.runner.RunWith -import kotlin.test.assertEquals - -@RunWith(JUnitParamsRunner::class) -class SerializeNullConfigTest { - private fun makeConfig(serializeNull: String) = com.amazon.ionhiveserde.configuration.SerializeNullConfig( - MapBasedRawConfiguration(mapOf("ion.serialize_null" to serializeNull))) - - private fun serializeNullOptions() = com.amazon.ionhiveserde.configuration.SerializeNullStrategy.values() - - @Test - @Parameters(method = "serializeNullOptions") - fun serializeNull(serializeNullStrategy: com.amazon.ionhiveserde.configuration.SerializeNullStrategy) { - val subject = makeConfig(serializeNullStrategy.name) - - assertEquals(serializeNullStrategy, subject.serializeNull) - } - - @Test - fun defaultSerializeNull() { - val subject = com.amazon.ionhiveserde.configuration.SerializeNullConfig(MapBasedRawConfiguration(mapOf())) - - assertEquals(com.amazon.ionhiveserde.configuration.SerializeNullStrategy.OMIT, subject.serializeNull) - } - - @Test(expected = IllegalArgumentException::class) - fun invalidSerializeNull() { - makeConfig("invalid option") - } -} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 7c97678..3256e3a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,19 +1,8 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 -include ':serde' +include ':hive2' +include ':hive3' include ':integration-test' +include ':hive-common' -enableFeaturePreview('STABLE_PUBLISHING') // gradle 5.0 compatibility