Add tests for new method and to check logs content

apache · Oct 27, 2023 · 1253d18 · 1253d18
1 parent d8215af
commit 1253d18
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 1 deletion.
diff --git a/pom.xml b/pom.xml
@@ -927,6 +927,7 @@
               <test.build.data>${test.build.data}</test.build.data>
               <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
               <tez.hadoop.version>${hadoop.version}</tez.hadoop.version>
+              <project.build.directory>${project.build.directory}</project.build.directory>
             </systemPropertyVariables>
           </configuration>
         </plugin>

diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -24,11 +24,17 @@
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.BitSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.UserPayload;
@@ -41,6 +47,7 @@
 import com.google.protobuf.ByteString;
 
 public class TestTezUtils {
+  private static final Path LOG_PATH = Paths.get(System.getProperty("project.build.directory"), "tez-test.log");
 
   @Test (timeout=2000)
   public void testByteStringToAndFromConf() throws IOException {
@@ -103,6 +110,18 @@ public void testByteStringAddToLargeConf() throws IOException {
     Assert.assertEquals(conf.get("testLargeValue"), largeValue);
   }
 
+  @Test public void testByteStringFromConfEmitsLogForLargeEntry() throws IOException {
+    Configuration conf = new Configuration(false);
+    conf.set(TezConfiguration.TEZ_LOGGING_PROPERTY_SIZE_THRESHOLD, "10");
+    conf.set(TezConfiguration.TEZ_LOGGING_PROPERTY_MASK, "false");
+    conf.set("tez.fake.property.id00", "ABCDEFGHIJK");
+    TezUtils.createByteStringFromConf(conf);
+    List<String> logLines = Lists.reverse(Files.readAllLines(LOG_PATH));
+    assertEquals("Entry 'tez.fake.property.id00' is unusually big (11 bytes); large entries may lead to OOM.",
+        logLines.get(1));
+    assertEquals("Large entry 'tez.fake.property.id00': ABCDEFGHIJK", logLines.get(0));
+  }
+
   @Test (timeout=2000)
   public void testPayloadToAndFromConf() throws IOException {
     Configuration conf = getConf();
@@ -294,4 +313,23 @@ public void testPopulateConfProtoFromEntries() {
       assertEquals(confBuilder.getConfKeyValuesList().size(), 1);
   }
 
+  @Test public void testPopulateConfProtoFromMapSetsKeyValuePair() {
+    DAGProtos.ConfigurationProto.Builder builder = DAGProtos.ConfigurationProto.newBuilder();
+    TezUtils.populateConfProtoFromMap(ImmutableMap.of("tez.fake.property", "someValue"), builder);
+    assertEquals(builder.getConfKeyValues(0).getKey(), "tez.fake.property");
+    assertEquals(builder.getConfKeyValues(0).getValue(), "someValue");
+  }
+
+  @Test public void testPopulateConfProtoFromMapEmitsLogForLargeEntry() throws IOException {
+    Map<String, String> map = new HashMap<>();
+    map.put(TezConfiguration.TEZ_LOGGING_PROPERTY_SIZE_THRESHOLD, "10");
+    map.put(TezConfiguration.TEZ_LOGGING_PROPERTY_MASK, "false");
+    map.put("tez.fake.property.id01", "ABCDEFGHIJK");
+    TezUtils.populateConfProtoFromMap(map, DAGProtos.ConfigurationProto.newBuilder());
+    List<String> logLines = Lists.reverse(Files.readAllLines(LOG_PATH));
+    assertEquals("Entry 'tez.fake.property.id01' is unusually big (11 bytes); large entries may lead to OOM.",
+        logLines.get(1));
+    assertEquals("Large entry 'tez.fake.property.id01': ABCDEFGHIJK", logLines.get(0));
+  }
+
 }
diff --git a/tez-common/src/test/resources/log4j.properties b/tez-common/src/test/resources/log4j.properties
@@ -12,8 +12,11 @@
 
 # log4j configuration used during build and unit tests
 
-log4j.rootLogger=info,stdout
+log4j.rootLogger=info,stdout,r
 log4j.threshhold=ALL
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.r=org.apache.log4j.RollingFileAppender
+log4j.appender.r.File=${project.build.directory}/tez-test.log
+log4j.appender.r.layout=org.apache.log4j.PatternLayout