log4j2.properties

mstatus = error
name = PropertiesConfig

# Define appenders
appender.console.type = Console
appender.console.name = ConsoleAppender
appender.console.target = SYSTEM_ERR
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

# Set root logger level
rootLogger.level = error
rootLogger.appenderRef.stdout.ref = ConsoleAppender

# Set PySpark logging level to INFO to see PySpark-specific logs
logger.pyspark.name = org.apache.spark.api.python
logger.pyspark.level = debug
logger.pyspark.appenderRef.console.ref = ConsoleAppender

# Set Spark SQL logging level to INFO for visibility into Spark SQL operations
logger.sparksql.name = org.apache.spark.sql.execution
logger.sparksql.level = info
logger.sparksql.appenderRef.console.ref = ConsoleAppender

# Optionally, adjust the logging level for other Spark components as needed
logger.sparkcontext.name = org.apache.spark.SparkContext
logger.sparkcontext.level = info
logger.sparkcontext.appenderRef.console.ref = ConsoleAppender