Snap 3307

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/arrayTypes.conf

@@ -4,22 +4,29 @@ as well as use the dataFrame API";
 
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf;
 
-TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
            io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.ArrayType
            io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_ArrayType_Validation"
            io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer ="dataFilesLocation=${dataFilesLocation}"
            io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
-TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
            io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.ArrayTypeAPI
            io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_ArrayType_Validation_Via_API"
            io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer ="dataFilesLocation=${dataFilesLocation}"
            io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
+           io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.ArrayTypeNULLValue
+           io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_ArrayTypeNULLValue_Validation"
+           io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+           threadGroups = snappyStoreThreads
+           maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/mapTypes.conf

@@ -20,6 +20,13 @@ TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
+           io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.MapTypeNULLValue
+           io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_MapTypeNULLValue_Validation"
+           io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+           threadGroups = snappyStoreThreads
+           maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/smartConnectorArrayTypes.conf

@@ -19,6 +19,12 @@ TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
+TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob
+             io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.complexdatatypes.SmartConnectorArrayTypeNULL
+             io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+             threadGroups = snappyStoreThreads
+             maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/smartConnectorMapTypes.conf

@@ -4,7 +4,6 @@ query as well as dataFrame API";
 
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf;
 
-
 TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob
            io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.complexdatatypes.SmartConnectorMapType
            io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation}"
@@ -19,6 +18,12 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_e
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob
+           io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.complexdatatypes.SmartConnectorMapTypeNULL
+           io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+           threadGroups = snappyStoreThreads
+           maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/smartConnectorStructTypes.conf

@@ -18,6 +18,12 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_e
            threadGroups = snappyStoreThreads
            maxTimesToRun = 1;
 
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob
+           io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.complexdatatypes.SmartConnectorStructTypeNULL
+           io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+           threadGroups = snappyStoreThreads
+          maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/java/io/snappydata/hydra/complexdatatypes/structTypes.conf

@@ -4,7 +4,6 @@ query as well as dataFrame API";
 
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf;
 
-
 TASK   taskClass   = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
              io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.StructType
              io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_StructType_Validation"
@@ -21,6 +20,13 @@ TASK   taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask
              threadGroups = snappyStoreThreads
              maxTimesToRun = 1;
 
+TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob
+           io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.complexdatatypes.StructTypeNULLValue
+           io.snappydata.hydra.cluster.SnappyPrms-userAppName = "ComplexType_StructTypeNULLValue_Validation"
+           io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar
+           threadGroups = snappyStoreThreads
+           maxTimesToRun = 1;
+
 INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf;
 hydra.Prms-maxResultWaitSec = 7200;
 hydra.Prms-totalTaskTimeSec = 7200;

dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/AllMixedTypes.scala

@@ -37,6 +37,7 @@ class AllMixedTypes extends SnappySQLJob{
     val sc : SparkContext = SparkContext.getOrCreate()
     val sqlContext : SQLContext = SQLContext.getOrCreate(sc)
     val printContent : Boolean = false
+    val isExecute : Boolean = false
 
     def getCurrentDirectory : String = new File(".").getCanonicalPath
     val outputFile : String = "ValidateAllMixedTypes" + "_" + "column" +
@@ -53,11 +54,17 @@ class AllMixedTypes extends SnappySQLJob{
     snc.sql("CREATE TABLE IF NOT EXISTS TwentyTwenty USING COLUMN " +
       "AS (SELECT * FROM TempTwenty)")
 
+    /**
+      * No need to execute below queries because validation routine runs it.
+      * Keep it here for ready reference.
+      */
+    if(isExecute) {
     snc.sql(ComplexTypeUtils.Mixed_Q1)
     snc.sql(ComplexTypeUtils.Mixed_Q2)
     snc.sql(ComplexTypeUtils.Mixed_Q3)
     snc.sql(ComplexTypeUtils.Mixed_Q4)
     snc.sql(ComplexTypeUtils.Mixed_Q5)
+    }
 
     if(printContent) {
       println("snc : Mixed_Q1 " + (snc.sql(ComplexTypeUtils.Mixed_Q1).show))

dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayType.scala

@@ -44,6 +44,7 @@ class ArrayType extends SnappySQLJob {
     val sc = SparkContext.getOrCreate()
     val sqlContext = SQLContext.getOrCreate(sc)
     val printContent : Boolean = false
+    val isExecute : Boolean = false
 
     /* --- Snappy Job --- */
     snc.sql("DROP TABLE IF EXISTS Student")
@@ -54,12 +55,18 @@ class ArrayType extends SnappySQLJob {
         "OPTIONS(path  '" + dataLocation + "')")
     snc.sql("CREATE TABLE Student USING COLUMN AS (SELECT * FROM TempArray)")
 
+    /**
+      * No need to execute below queries because validation routine runs it.
+      * Keep it here for ready reference.
+      */
+    if(isExecute) {
     snc.sql(ComplexTypeUtils.Array_Q1)
     snc.sql(ComplexTypeUtils.Array_Q2)
     snc.sql(ComplexTypeUtils.Array_Q3)
     snc.sql(ComplexTypeUtils.Array_View)
     snc.sql(ComplexTypeUtils.Array_Q4)
     snc.sql(ComplexTypeUtils.Array_Q5)
+    }
 
     if(printContent) {
       println(snc.sql(ComplexTypeUtils.Array_Q1).show())

dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayTypeNULLValue.scala

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package io.snappydata.hydra.complexdatatypes
+
+import java.io.{File, FileOutputStream, PrintWriter}
+
+import com.typesafe.config.Config
+import io.snappydata.hydra.SnappyTestUtils
+import org.apache.spark.sql._
+
+class ArrayTypeNULLValue extends SnappySQLJob {
+  override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid()
+
+  override def runSnappyJob(snappySession: SnappySession, jobConfig: Config): Any = {
+    // scalastyle:off println
+    println("Validation for NULL Value in ArraysType column Job started...")
+    val snc : SnappyContext = snappySession.sqlContext
+    val spark : SparkSession = SparkSession.builder().enableHiveSupport().getOrCreate()
+    val sqlContext = spark.sqlContext
+    val outputFile = "ValidateArrayTypeNULLValue" + "_"  +
+      System.currentTimeMillis() + jobConfig.getString("logFileName")
+    val pw : PrintWriter = new PrintWriter(new FileOutputStream(new File(outputFile), false))
+
+    /**
+      *  Test : NULL value in Array Type column.
+      */
+    snc.sql(ComplexTypeUtils.createSchemaST)
+    spark.sql(ComplexTypeUtils.createSchemaST)
+    /**
+      * Test Case 1 : ArrayType Column is last column in the table.
+      */
+    snc.sql(ComplexTypeUtils.createTableLastColumnArrayType)
+    spark.sql(ComplexTypeUtils.createTableInSparkArrTypeLastColumn)
+    snc.sql(ComplexTypeUtils.insertNullInLastColumn)
+    spark.sql(ComplexTypeUtils.insertNullInLastColumn)
+    snc.sql(ComplexTypeUtils.insertNormalDataLastColumn)
+    spark.sql(ComplexTypeUtils.insertNormalDataLastColumn)
+    /**
+      *  Test Case 2 : ArrayType Column is between (say middle)  the other data types in the table.
+      */
+    snc.sql(ComplexTypeUtils.createTableMiddleColumnArrayType)
+    spark.sql(ComplexTypeUtils.createTableInSparkArrayTypeMiddleColumn)
+    snc.sql(ComplexTypeUtils.insertNullInMiddleColumn)
+    spark.sql(ComplexTypeUtils.insertNullInMiddleColumn)
+    snc.sql(ComplexTypeUtils.insertNormalDataMiddleColumn)
+    spark.sql(ComplexTypeUtils.insertNormalDataMiddleColumn)
+    /**
+      *  Test Case 3: ArrayType Column is the first column in the table.
+     */
+    snc.sql(ComplexTypeUtils.createTableFirstColumnArrayType)
+    spark.sql(ComplexTypeUtils.createTableInSparkArrayTypeFirstColumn)
+    snc.sql(ComplexTypeUtils.insertNullInFirstColumn)
+    spark.sql(ComplexTypeUtils.insertNullInFirstColumn)
+    snc.sql(ComplexTypeUtils.insertNormalDataFirstColumn)
+    spark.sql(ComplexTypeUtils.insertNormalDataFirstColumn)
+    /**
+      * Validation Routine
+      */
+    SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.selectLastColumn,
+      "AQ1", "column", pw, sqlContext)
+    pw.println("-- Insertion of NULL value in ArrayType last column OK --")
+    pw.flush()
+    SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.selectMiddleColumn,
+      "AQ2", "column", pw, sqlContext)
+    pw.println("-- Insertion of NULL value in ArrayType middle column OK --")
+    pw.flush()
+    SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.selectFirstColumn,
+      "AQ3", "column", pw, sqlContext)
+    pw.println("-- Insertion of NULL value in ArrayType first column OK --")
+    pw.flush()
+    pw.close()
+
+    snc.sql(ComplexTypeUtils.dropTableStudentLast)
+    snc.sql(ComplexTypeUtils.dropTableStudentMiddle)
+    snc.sql(ComplexTypeUtils.dropTableStudentMiddle)
+    spark.sql(ComplexTypeUtils.dropTableStudentLast)
+    spark.sql(ComplexTypeUtils.dropTableStudentMiddle)
+    spark.sql(ComplexTypeUtils.dropTableStudentLast)
+    snc.sql(ComplexTypeUtils.dropDatabaseST)
+    spark.sql(ComplexTypeUtils.dropDatabaseST)
+  }
+}

dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStringInMapAsValue.scala

@@ -43,6 +43,7 @@ class ArraysOfStringInMapAsValue extends SnappySQLJob{
     val pw : PrintWriter = new PrintWriter(new FileOutputStream(new File(outputFile), false))
     val dataLocation = jobConfig.getString("dataFilesLocation")
     val printContent : Boolean = false
+    val isExecute : Boolean = false
 
 
     /* --- Snappy Job --- */
@@ -56,10 +57,16 @@ class ArraysOfStringInMapAsValue extends SnappySQLJob{
     val fp = snc.sql(  "CREATE TABLE IF NOT EXISTS FamousPeople " +
       "USING COLUMN AS (SELECT * FROM TempFamousPeople)")
 
+    /**
+      * No need to execute below queries because validation routine runs it.
+      * Keep it here for ready reference.
+      */
+    if(isExecute) {
     snc.sql(ComplexTypeUtils.Array_Map_TempView)
     snc.sql(ComplexTypeUtils.Array_Map_Q1)
     snc.sql(ComplexTypeUtils.Array_Map_Q2)
     snc.sql(ComplexTypeUtils.Array_Map_Q3)
+    }
 
     if(printContent) {
       println("snc : Array_Map_Q1 " + snc.sql(ComplexTypeUtils.Array_Map_Q1).show)

dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStructType.scala

@@ -41,6 +41,9 @@ class ArraysOfStructType extends SnappySQLJob{
     val pw : PrintWriter = new PrintWriter(new FileOutputStream(new File(outputFile), false))
     val dataLocation = jobConfig.getString("dataFilesLocation")
     val printContent : Boolean = false
+    //  Keep the isExecute value true because in validation
+    // routine there is a Hydra Exception which needs to be fixed.
+    val isExecute : Boolean = true
 
     /* --- Snappy Job --- */
     snc.sql("DROP TABLE IF EXISTS TwoWheeler")
@@ -51,11 +54,13 @@ class ArraysOfStructType extends SnappySQLJob{
     snc.sql("CREATE TABLE IF NOT EXISTS TwoWheeler USING COLUMN " +
       "AS (SELECT * FROM TempBike)")
 
+    if(isExecute) {
     snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1)
     snc.sql(ComplexTypeUtils.ArraysOfStruct_Q2)
     snc.sql(ComplexTypeUtils.ArraysOfStruct_Q3)
     snc.sql(ComplexTypeUtils.ArraysOfStruct_Q4)
     snc.sql(ComplexTypeUtils.ArraysOfStruct_Q5)
+    }
 
     if(printContent) {
       println("snc : ArraysOfStruct_Q1  " + (snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1).show()))