build.gradle

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
plugins {
    id "org.nosphere.apache.rat" version "0.3.0"
}
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'checkstyle'
apply plugin: 'jacoco'
apply plugin: 'findbugs'
apply plugin: 'maven-publish'

apply from: 'gradle/sqoop-version-gen.gradle'
apply from: 'gradle/sqoop-package.gradle'

group = 'org.apache.sqoopgradle'

sourceCompatibility = javaSourceCompatibilityVersion

repositories {
    mavenLocal()
    maven {
        url 'https://repo1.maven.org/maven2/'
    }
    maven {
        url 'http://maven.cubrid.org/'
    }

    // DB2 driver repository
    maven {
        url 'https://artifacts.alfresco.com/nexus/content/repositories/public/'
    }

    // Oracle driver repository
    maven {
        url 'http://www.datanucleus.org/downloads/maven2/'
    }
    maven {
        url 'https://maven.aliyun.com/repository/spring-plugin'
    }
    maven {
        url 'http://repo1.acceldata.dev/repository/odp-central'
    }
}

sourceSets {
    main {
        java {
            srcDirs = ['src/java']
        }
        resources {
            srcDirs = ['src/java']
        }
    }
    test {
        java {
            srcDirs = ['src/test']
            exclude 'aop/'
        }
        resources {
            srcDirs = ['src/test', 'testdata/hcatalog/conf', 'conf']
            exclude 'aop/'
        }
    }
    aop {
        java {
            srcDirs = ['src/test/aop']
        }
        resources {
            srcDirs = ['src/test/aop']
        }
    }
}

configurations {
    common.extendsFrom(redist)
    compile.extendsFrom(common)
}

compileJava.dependsOn(SqoopVersionFileGen)

compileJava {
    options.compilerArgs += '-proc:none'
    options.compilerArgs += '-Xlint:all'
}

compileTestJava {
    options.compilerArgs += '-proc:none'
    options.compilerArgs += '-Xlint:all'
}

configurations.all {
    resolutionStrategy {
        force group: 'org.apache.avro', name: 'avro', version: avroVersion
        force group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
        force group: 'com.google.guava', name: 'guava', version: guavaVersion
        force group: 'com.google.protobuf', name: 'protobuf-java', version: hiveProtobufVersion
    }
    exclude group: 'org.apache.hadoop', module: 'avro'
}

def forkEveryDefault = Integer.valueOf(System.getProperty("forkEvery.default", "0"))
def ignoreTestFailures = Boolean.getBoolean("ignoreTestFailures")

dependencies {
    compile group: 'org.apache.hadoop', name: 'hadoop-aws', version: hadoopVersion
    compile (group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion) {
        exclude group: "asm", module: "asm"
    }
    compile (group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion) {
        exclude group: "asm", module: "asm"
    }
    compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion) {
        exclude group: "asm", module: "asm"
    }
    compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion) {
        exclude group: "asm", module: "asm"
    }
    compile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion

    aopCompile group: 'org.aspectj', name: 'aspectjtools', version: aspectjVersion
    aopCompile group: 'org.aspectj', name: 'aspectjrt', version: aspectjVersion
    aopCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion

    common group: 'org.apache.accumulo', name: 'accumulo-core', version: accumuloVersion
    common group: 'org.apache.accumulo', name: 'accumulo-minicluster', version: accumuloVersion

    common group: 'org.eclipse.jetty', name: 'jetty-runner', version: jettyVersion

    common group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion
    common group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion
    common group: 'org.apache.hbase', name: 'hbase-client', version: hbaseVersion
    common group: 'org.apache.hbase', name: 'hbase-common', version: hbaseVersion
    common(group: 'org.apache.hive.hcatalog', name: 'hive-hcatalog-core', version: hcatalogVersion) {
        exclude group: 'org.apache.avro', module: 'avro'
        exclude group: 'org.apache.hive', module: 'hive-exec'
    }
    common(group: 'org.apache.hive', name: 'hive-exec', version: hcatalogVersion, classifier: 'core') {
        exclude group :  'org.apache.calcite', module :'calcite-core'
    }
    // Kryo and calcite are dependencies of hive-exec:core
    common group: 'com.esotericsoftware', name: 'kryo', version: kryoVersion
    common group: 'org.apache.calcite', name: 'calcite-core', version: calciteVersion
    common(group: 'org.apache.hive', name: 'hive-jdbc', version: hcatalogVersion) {
        exclude group: 'org.apache.avro', module: 'avro'
        exclude group: 'asm', module: 'asm'
    }
    common group: 'commons-cli', name: 'commons-cli', version: commonscliVersion
    common group: 'commons-logging', name: 'commons-logging', version: commonsloggingVersion
    common group: 'commons-net', name: 'commons-net', version: commonsnetVersion
    common group: 'log4j', name: 'log4j', version: log4jVersion
    common group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion
    common group: 'org.apache.parquet', name: 'parquet-hadoop-bundle', version: parquetVersion

    testCompile group: 'com.h2database', name: 'h2', version: h2Version
    testCompile group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion, classifier: 'tests'
    testCompile group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion, classifier: 'tests'
    testCompile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion, classifier: 'tests'
    testCompile (group: 'org.apache.hadoop', name: 'hadoop-minikdc', version: hadoopVersion) {
        exclude group: 'org.apache.directory.api', module: 'api-ldap-schema-data'
    }
    testCompile group: 'junit', name: 'junit', version: junitVersion
    testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
    testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
    testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
    testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'

    // JDBC drivers
    testCompile group: 'mysql', name: 'mysql-connector-java', version: mySqlConnectorVersion
    testCompile group: 'cubrid', name: 'cubrid-jdbc', version: cubridJdbcDriverVersion
    testCompile group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: msSqlJdbcDriverVersion
    testCompile group: 'com.ibm.db2', name: 'jcc', version: db2JdbcDriverVersion
    testCompile group: 'oracle', name: 'ojdbc6', version: oracleJdbcDriverVersion
}

task unitTest (type: Test) {
    description 'Run unit tests which do not need proprietary JDBC driver'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
        excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
    }
}

task integrationTest(type: Test) {
    description 'Run integration tests which do not need a docker container or an external database/service'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
        excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
    }
}

task thirdPartyTest (type: Test) {
    description 'Run Third-party Tests - you need to specify -Dsqoop.thirdparty.lib.dir where the Third party driver ' +
            'jars reside (relative to the project directory)'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
    systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
    systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
    systemProperty "ms.datatype.test.data.file.delim" ,","
    systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
}

task postgreSqlTest (type: Test) {
    description 'Run Postgresql third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task mySqlTest (type: Test) {
    description 'Run MySql third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MysqlTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task cubridTest (type: Test) {
    description 'Run Cubrid third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.CubridTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task db2Test (type: Test) {
    description 'Run DB2 third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.Db2Test'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task mainFrameTest (type: Test) {
    description 'Run Mainframe third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MainFrameTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task oracleTest (type: Test) {
    description 'Run Oracle third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task oracleXeTest (type: Test) {
    description 'Run Oracle third party tests which can be run with Oracle Express Edition too.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
        excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task oracleEeTest (type: Test) {
    description 'Run Oracle third party tests which require Oracle Enterprise Edition.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
}

task sqlServerTest (type: Test) {
    description 'Run MS SqlServer third party tests.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
    systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
    systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
    systemProperty "ms.datatype.test.data.file.delim" ,","
    systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
}

test {
    description 'Run tests that do not need external JDBC driver and/or a docker container. ' +
            'This is the same as running unitTest, integrationTest and kerberizedTest.'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
        excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
    }
}

task s3Test(type: Test) {
    description 'Run S3 tests'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.S3Test'
    }
}

task allTest (type: Test){
    description 'Run all Sqoop tests'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
        includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
        includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
        excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
        excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
    }
}

def testBuildDir = "$buildDir/test/"
def testBuildDirData ="$testBuildDir/data/"

tasks.withType(Test) {
    doFirst {
        project.mkdir(testBuildDir)
        project.mkdir(testBuildDirData)
    }

    testLogging {
        events 'passed', 'skipped'
    }
    workingDir = testBuildDirData
    systemProperties(System.getProperties())
    systemProperty "test.data.dir", "$projectDir/testdata"
    systemProperty 'test.build.data', "$testBuildDir/data"
    systemProperty "hadoop.root.logger", "DEBUG,console"
    systemProperty "user.dir", workingDir
    systemProperty "sqoop.src.dir", "$testBuildDir/data"
    systemProperty "hadoop.tmp.dir", "$testBuildDir/hadoop"
    systemProperty "fs.default.name", ""
    systemProperty "hadoop.log.dir", "$testBuildDir/logs"
    systemProperty "hive.home", "$projectDir/testdata/hive"
    systemProperty "sqoop.throwOnError", ""

    minHeapSize = "512m"
    maxHeapSize = "8G"
    jvmArgs "-da:org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge"

    jacoco{
        excludes = ["**/SqoopVersion*"]
    }

    forkEvery forkEveryDefault
    ignoreFailures ignoreTestFailures
}

task kerberizedTest (type: Test){
    description 'Run Kerberized Test'
    // A Gradle test task with forkEvery 1 and includeCategories performs poorly because it starts a new JVM even for the filtered tests.
    // To work around this performance problem we need to add every kerberized test in a separate include field here.
    include '**/TestKerberosAuthenticator*'
    include '**/HBaseKerberizedConnectivityTest*'
    include '**/TestHiveMiniCluster*'
    include '**/TestHiveServer2TextImport*'
    useJUnit {
        includeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
    }
    forkEvery 1
    ignoreFailures ignoreTestFailures
}

test.finalizedBy(kerberizedTest)
allTest.finalizedBy(kerberizedTest)

tasks.withType(Checkstyle) {
    reports {
        xml.enabled false
        html.enabled true
        html.stylesheet resources.text.fromFile('config/checkstyle/checkstyle-noframes.xsl')
    }
    ignoreFailures = true
}

tasks.withType(FindBugs) {
    reports {
        xml.enabled false
        html.enabled true
    }
    ignoreFailures = false
}

findbugs {
    sourceSets = []
}

checkstyle {
    sourceSets = []
    toolVersion checkstyleVersion
}

jacocoTestReport {
    description "Generates a coverage report of the unit test results under $buildDir/jacocoHtml/test/index.html"
    executionData test, kerberizedTest
    reports {
        xml.enabled true
        csv.enabled false
        html.destination file("$buildDir/jacocoHtml/test")
    }
}

task jacocoThirdPartyReport(type: JacocoReport){
    description "Generates a coverage report of the thirdparty test results under $buildDir/jacocoHtml/3rd/index.html"
    sourceSets sourceSets.main
    executionData thirdPartyTest
    reports {
        xml.enabled true
        csv.enabled false
        html.destination file("$buildDir/jacocoHtml/3rd")
    }
}

task jacocoCompositeReport(type: JacocoReport){
    description "Generates a composite coverage report of test and thirdparty test results under $buildDir/jacocoHtml/index.html"
    sourceSets sourceSets.main
    executionData test, kerberizedTest, thirdPartyTest
    reports {
        xml.enabled true
        csv.enabled false
        html.destination file("$buildDir/jacocoHtml/")
    }
}

javadoc {
    source = sourceSets.main.allJava
    include 'com/cloudera/sqoop/lib/**'
    include 'org/apache/sqoop/lib/**'
    destinationDir = reporting.file("${project.docsDir}/api")
}

task faultInjectionJar(type: Jar) {
    description 'Create the test jar'
    appendix = 'test'
    from sourceSets.aop.output
}

def archiveName = jar.baseName + "-" + jar.version
def binArtifactName = archiveName + ".bin__hadoop-" + hadoopVersion
def distDir = "$buildDir/$binArtifactName"

rat {
    inputDir = distDir
    reportDir = project.file(buildDir)
    excludes = ['**/docs/**', '**/testdata/**', '**/.idea/**', '**/test-dir/**', '**/.settings/**', '**/.gradle/**','**/gradle/**' ,'**/gradle/build/**', '*.iml', '*.iws', '*.ipr', 'derby.log', '**/gradlew**' ]
    failOnError = true
    xmlOutput = false
    htmlOutput = true
    plainOutput = false
}
rat.dependsOn('packageDist')

def scripts = ['export', 'list-databases', 'metastore', 'create-hive-table', 'help',
               'import-mainframe', 'list-tables', 'version',
               'eval', 'import', 'job', 'merge',
               'import-all-tables', 'codegen']


task runSqoopHelp(type: JavaExec, dependsOn: 'classes') {
    main = 'org.apache.sqoop.Sqoop'
    jvmArgs = ['-Dhadoop.security.log.file=./build/security-audit.log']
    classpath = sourceSets.main.runtimeClasspath + configurations.runtime
    args = ['help']
    standardOutput = new ByteArrayOutputStream()
    ext.helpOut = {
        standardOutput.toString()
    }

}

task createAllStartScripts(dependsOn: 'runSqoopHelp' ) doLast{
    // Placeholder task with validation
    // We call Sqoop help to verify that all tasks that we create scripts for are present in Sqoop help and vice-versa
    // We currently depend on the format of the help output to parse the relevant information out
    def matcher = (tasks.runSqoopHelp.helpOut() =~/(?m)^\s\s([a-z-]+)\s+\w+/)
    def matches = new String[matcher.getCount()]
    for (def i=0; i < matcher.getCount(); i++) {
        matches[i] = matcher[i][1]
    }
    if (matches.sort() != scripts.sort())
        throw new GradleException('Error: script list in gradle build script and result of Sqoop help are not the same, but should be')
}

scripts.each() { scriptName ->
    def t = tasks.create(name: scriptName + 'StartScript', type: CreateStartScripts) {
        unixStartScriptGenerator.template = resources.text.fromFile('gradle/customUnixStartScript.txt')
        windowsStartScriptGenerator.template = resources.text.fromFile('gradle/customWindowsStartScript.txt')
        outputDir = file("$buildDir/bin")
        applicationName = scriptName
        mainClassName = 'com.cloudera.sqoop.Sqoop'
        classpath = project.configurations.runtime
        doLast {
            file("$buildDir/bin/"+ scriptName).renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName))
            file("$buildDir/bin/"+ scriptName +".bat").renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName+".bat"))
        }
    }
    t.dependsOn('compileJava')
    createAllStartScripts.dependsOn(t)
}

task checkVersion doLast {
    if (version.contains('SNAPSHOT'))
        throw new GradleException('Error: cannot release a snapshot. Set -Pversion')
}

task release(dependsOn: ['checkVersion', 'tar', 'rat']) {
    doLast {
        println 'Release complete'
        println "Binary tar: $buildDir/$binArtifactName-${version}.tar.gz"
        println "Documentation: $buildDir/docs"
        println "Release notes: $buildDir/docs/sqoop-$version" + ".releasenotes.html"
        println "Release audit report: $buildDir/index.html"
    }
}

task relnotes(type: Exec)  {
    workingDir 'src/scripts'
    if (!version.contains('SNAPSHOT')) {
        commandLine "python", "relnotes.py", "$buildDir/docs", "$projectDir", "$oldHash\\..HEAD", "$version", "$oldVersion"
    } else {
        commandLine "true" //noop
        println "Will not run releasenotes for SNAPSHOT version."
    }
}

task realDocs(type: Exec, dependsOn: ['relnotes', 'javadoc']) {
    workingDir projectDir
    commandLine "make", "-C", "$projectDir/src/docs", "BUILDROOT=$buildDir", "VERSION=$version"
}

task docs(dependsOn: ['realDocs', 'relnotes', 'javadoc']) {
    //placeholder
}
def $mvnRepo = System.getProperty("mvnRepo", "snapshots")

publishing {
    repositories {
        maven {
            name "cloudera." + $mvnRepo + ".repo"
            url "https://repository.cloudera.com/content/repositories/"+$mvnRepo
            credentials {
                username 'username'
                password 'password'
            }
        }
    }
}
publishing {
    publications {
        sqoop(MavenPublication) {
            groupId group
            version  version
            from components.java
            artifact sourceJar {
                classifier "sources"
            }
        }
        sqoopTest(MavenPublication) {
            groupId group
            version  version
            artifactId 'sqoop-test'
            artifact testJar
            artifact sourceTestJar
            pom.withXml {
                def dependencies = asNode().appendNode('dependencies')
                configurations.testRuntime.getResolvedConfiguration().getFirstLevelModuleDependencies().each {
                    def dependency = dependencies.appendNode('dependency')
                    dependency.appendNode('groupId', it.moduleGroup)
                    dependency.appendNode('artifactId', it.moduleName)
                    dependency.appendNode('version', it.moduleVersion)
                }
            }
        }
    }
}

wrapper {
    gradleVersion = '4.9'
}