From 5f5c15dd23393f8ca080a1e462f5bdc5a0b9b635 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 14 Jul 2021 09:17:07 +0530 Subject: [PATCH] Reduced logging by BlockManager in default log4j.properties also fixed a putInto hydra test and update spark module link --- cluster/conf/log4j.properties.template | 8 ++++++++ .../src/main/scala/io/snappydata/impl/LeadImpl.scala | 10 ++++------ .../scripts/putInto/create_and_load_testLTable.sql | 2 +- .../io/snappydata/hydra/putInto/ConcPutIntoTest.scala | 9 ++++++--- spark | 2 +- store | 2 +- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cluster/conf/log4j.properties.template b/cluster/conf/log4j.properties.template index 3bc8aaa561..341ff50b3f 100644 --- a/cluster/conf/log4j.properties.template +++ b/cluster/conf/log4j.properties.template @@ -99,6 +99,7 @@ log4j.logger.org.apache.spark.scheduler.DAGScheduler=WARN log4j.logger.org.apache.spark.scheduler.TaskSetManager=WARN log4j.logger.org.apache.spark.scheduler.FairSchedulableBuilder=WARN log4j.logger.org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint=WARN +log4j.logger.org.apache.spark.storage.BlockManager=WARN log4j.logger.org.apache.spark.storage.BlockManagerInfo=WARN log4j.logger.org.apache.hadoop.hive=WARN log4j.logger.org.apache.spark.sql.execution.datasources=WARN @@ -116,6 +117,8 @@ log4j.logger.org.datanucleus=ERROR # Task logger created in SparkEnv log4j.logger.org.apache.spark.Task=WARN log4j.logger.org.apache.spark.sql.catalyst.parser.CatalystSqlParser=WARN +# HiveExternalCatalog spits out a warning every time a non-hive table is persisted in meta-store +log4j.logger.org.apache.spark.sql.hive.SnappyHiveExternalCatalog=ERROR # Keep log-level of some classes as INFO even if root level is higher log4j.logger.io.snappydata.impl.LeadImpl=INFO @@ -123,6 +126,11 @@ log4j.logger.io.snappydata.impl.ServerImpl=INFO log4j.logger.io.snappydata.impl.LocatorImpl=INFO log4j.logger.spray.can.server.HttpListener=INFO +# Note: all code generation classes that dump using "code" logger should +# also be listed in ClientSharedUtils.initLog4j for removal in case top-level +# file has not been set (e.g. common for JDBC clients) else an empty +# generatedcode.log will be created. + # for generated code of plans log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=INFO, code log4j.additivity.org.apache.spark.sql.execution.WholeStageCodegenExec=false diff --git a/cluster/src/main/scala/io/snappydata/impl/LeadImpl.scala b/cluster/src/main/scala/io/snappydata/impl/LeadImpl.scala index 77cf3b7ca0..423bf4bf99 100644 --- a/cluster/src/main/scala/io/snappydata/impl/LeadImpl.scala +++ b/cluster/src/main/scala/io/snappydata/impl/LeadImpl.scala @@ -26,6 +26,7 @@ import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} + import akka.actor.ActorSystem import com.gemstone.gemfire.CancelException import com.gemstone.gemfire.cache.CacheClosedException @@ -50,6 +51,7 @@ import org.apache.thrift.transport.TTransportException import spark.jobserver.JobServer import spark.jobserver.auth.{AuthInfo, SnappyAuthenticator, User} import spray.routing.authentication.UserPass + import org.apache.spark.sql.collection.{ToolsCallbackInit, Utils} import org.apache.spark.sql.execution.SecurityUtils import org.apache.spark.sql.hive.thriftserver.SnappyHiveThriftServer2 @@ -57,8 +59,6 @@ import org.apache.spark.sql.{SnappyContext, SnappySession} import org.apache.spark.util.LocalDirectoryCleanupUtil import org.apache.spark.{Logging, SparkCallbacks, SparkConf, SparkContext, SparkException} -import scala.collection.mutable.ArrayBuffer - class LeadImpl extends ServerImpl with Lead with ProtocolOverrides with Logging { @@ -336,10 +336,8 @@ class LeadImpl extends ServerImpl with Lead // If recovery mode then initialize the recovery service if (Misc.getGemFireCache.isSnappyRecoveryMode) { - if (enableTableCountInUI.equalsIgnoreCase("true")) - RecoveryService.collectViewsAndPrepareCatalog(true) - else - RecoveryService.collectViewsAndPrepareCatalog(false) + RecoveryService.collectViewsAndPrepareCatalog( + enableTableCountInUI.equalsIgnoreCase("true")) } if (jobServerWait) { diff --git a/dtests/src/resources/scripts/putInto/create_and_load_testLTable.sql b/dtests/src/resources/scripts/putInto/create_and_load_testLTable.sql index 094af71d03..4f54ec0a57 100644 --- a/dtests/src/resources/scripts/putInto/create_and_load_testLTable.sql +++ b/dtests/src/resources/scripts/putInto/create_and_load_testLTable.sql @@ -1,4 +1,4 @@ -- DROP TABLE IF ALREADY EXISTS -- DROP TABLE IF EXISTS testL; -create table testL (id long, data string, data2 decimal) using column options (partition_by 'id', key_columns 'id'); +create table testL (id long, data string, data2 decimal(38,10)) using column options (partition_by 'id', key_columns 'id'); diff --git a/dtests/src/test/scala/io/snappydata/hydra/putInto/ConcPutIntoTest.scala b/dtests/src/test/scala/io/snappydata/hydra/putInto/ConcPutIntoTest.scala index 578550cee0..5eaac1b914 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/putInto/ConcPutIntoTest.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/putInto/ConcPutIntoTest.scala @@ -26,10 +26,12 @@ import scala.concurrent.{Await, Future} object ConcPutIntoTest { - def concPutInto(primaryLocatorHost: String, primaryLocatorPort: String, numThreads: Integer): Any = { + def concPutInto(primaryLocatorHost: String, primaryLocatorPort: String, + numThreads: Integer): Any = { val globalId = new AtomicInteger() val doPut = () => Future { - val conn = DriverManager.getConnection("jdbc:snappydata://" + primaryLocatorHost + ":" + primaryLocatorPort) + val conn = DriverManager.getConnection( + "jdbc:snappydata://" + primaryLocatorHost + ":" + primaryLocatorPort) val stmt = conn.createStatement() val myId = globalId.getAndIncrement() val blockSize = 100000L @@ -60,7 +62,8 @@ object ConcPutIntoTest { queryTasks.foreach(Await.result(_, Duration.Inf)) } - def conSelect(primaryLocatorHost: String, primaryLocatorPort: String, numThreads: Integer): Any = { + def conSelect(primaryLocatorHost: String, primaryLocatorPort: String, + numThreads: Integer): Any = { val globalId = new AtomicInteger() val doQuery = () => Future { val conn = DriverManager.getConnection("jdbc:snappydata://localhost:1527") diff --git a/spark b/spark index 3d731893f9..451575e110 160000 --- a/spark +++ b/spark @@ -1 +1 @@ -Subproject commit 3d731893f9c6a661cf7c323e77d2369191f02674 +Subproject commit 451575e110c459082d2346d89cafbd2d689f14d9 diff --git a/store b/store index 1e529b7db0..0f546384b5 160000 --- a/store +++ b/store @@ -1 +1 @@ -Subproject commit 1e529b7db0d849a939c4b971214a4a3566329c65 +Subproject commit 0f546384b51c4018b5ce412962d720a8ffdb3773