Identify invalid IBDO data/array modifications.

src/main/java/emissary/core/BaseDataObject.java

@@ -182,6 +182,8 @@ public class BaseDataObject implements Serializable, Cloneable, Remote, IBaseDat
      */
     protected SeekableByteChannelFactory seekableByteChannelFactory;
 
+    protected final Map<byte[], String> arrayHashMap = new HashMap<>();
+
     protected enum DataState {
         NO_DATA, CHANNEL_ONLY, BYTE_ARRAY_ONLY, BYTE_ARRAY_AND_CHANNEL
     }
@@ -221,6 +223,21 @@ protected DataState getDataState() {
         }
     }
 
+    @Override
+    public void checkAndResetArrayHashMap(final String placeName) {
+        for (Map.Entry<byte[], String> e : arrayHashMap.entrySet()) {
+            if (!ByteUtil.sha256Bytes(e.getKey()).equals(e.getValue())) {
+                logger.warn("IBDO-DATA-MODIFICATION: Data array modified without setting in {}!", placeName);
+            }
+        }
+
+        arrayHashMap.clear();
+
+        if (this.theData != null) {
+            arrayHashMap.put(this.theData, ByteUtil.sha256Bytes(this.theData));
+        }
+    }
+
     /**
      * Create an empty BaseDataObject.
      */
@@ -327,6 +344,7 @@ public void setChannelFactory(final SeekableByteChannelFactory sbcf) {
         Validate.notNull(sbcf, "Required: SeekableByteChannelFactory not null");
         this.theData = null;
         this.seekableByteChannelFactory = sbcf;
+        this.arrayHashMap.clear();
     }
 
     /**
@@ -378,7 +396,11 @@ public byte[] data() {
                 return theData;
             case CHANNEL_ONLY:
                 // Max size here is slightly less than the true max size to avoid memory issues
-                return SeekableByteChannelHelper.getByteArrayFromBdo(this, MAX_BYTE_ARRAY_SIZE);
+                final byte[] bytes = SeekableByteChannelHelper.getByteArrayFromBdo(this, MAX_BYTE_ARRAY_SIZE);
+
+                arrayHashMap.put(bytes, ByteUtil.sha256Bytes(bytes));
+
+                return bytes;
             case NO_DATA:
             default:
                 return null; // NOSONAR maintains backwards compatibility
@@ -396,6 +418,9 @@ public void setData(@Nullable final byte[] newData) {
         } else {
             this.theData = newData;
         }
+
+        arrayHashMap.clear();
+        arrayHashMap.put(this.theData, ByteUtil.sha256Bytes(this.theData));
     }
 
     /**
@@ -422,6 +447,9 @@ public void setData(@Nullable final byte[] newData, final int offset, final int
             this.theData = new byte[length];
             System.arraycopy(newData, offset, this.theData, 0, length);
         }
+
+        arrayHashMap.clear();
+        arrayHashMap.put(this.theData, ByteUtil.sha256Bytes(this.theData));
     }
 
     /**

src/main/java/emissary/core/IBaseDataObject.java

@@ -26,6 +26,8 @@ static enum MergePolicy {
      */
     String DEFAULT_PARAM_SEPARATOR = ";";
 
+    void checkAndResetArrayHashMap(final String placeName);
+
     /**
      * Return the data as a byte array. If using a channel to the data, calling this method will only return up to
      * Integer.MAX_VALUE bytes of the original data.

src/main/java/emissary/place/ServiceProviderPlace.java

@@ -570,6 +570,7 @@ public List<IBaseDataObject> agentProcessHeavyDuty(IBaseDataObject payload) thro
         MDC.put(MDCConstants.SERVICE_LOCATION, this.getKey());
         try {
             List<IBaseDataObject> l = processHeavyDuty(payload);
+            payload.checkAndResetArrayHashMap(placeName);
             rehash(payload);
             return l;
         } catch (Exception e) {

src/test/java/emissary/core/BaseDataObjectTest.java

@@ -8,8 +8,10 @@
 import emissary.core.channels.SeekableByteChannelHelper;
 import emissary.directory.DirectoryEntry;
 import emissary.pickup.Priority;
+import emissary.test.core.junit5.LogbackTester;
 import emissary.test.core.junit5.UnitTest;
 
+import ch.qos.logback.classic.Level;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 import org.junit.jupiter.api.AfterEach;
@@ -25,6 +27,7 @@
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -1319,4 +1322,115 @@ void testExtractedRecordClone() {
             fail("Clone method should have been called", ex);
         }
     }
+
+    @Test
+    void testChannelFactoryInArrayOutNoSet() throws IOException {
+        final byte[] bytes = "These are the test bytes!".getBytes(StandardCharsets.US_ASCII);
+        final Level[] levels = new Level[] {Level.WARN};
+        final String[] messages = new String[] {"IBDO-DATA-MODIFICATION: Data array modified without setting in TestPlace!"};
+        final boolean[] throwables = new boolean[] {false};
+
+        try (LogbackTester logbackTester = new LogbackTester(BaseDataObject.class.getName())) {
+            final IBaseDataObject ibdo = new BaseDataObject();
+
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(bytes));
+
+            final byte[] data = ibdo.data();
+
+            Arrays.fill(data, (byte) 0);
+
+            ibdo.checkAndResetArrayHashMap("TestPlace");
+
+            assertArrayEquals(bytes, ibdo.data());
+            logbackTester.checkLogList(levels, messages, throwables);
+        }
+    }
+
+    final static String DATA_MODIFICATION_ERROR = "IBDO-DATA-MODIFICATION: Data array modified without setting in TestPlace!";
+    final static byte[] DATA_MODIFICATION_BYTES = "These are the test bytes!".getBytes(StandardCharsets.US_ASCII);
+
+    @Test
+    void testChannelFactoryInTwoArrayOutNoSet1() throws IOException {
+        final Level[] levels = new Level[] {Level.WARN};
+        final String[] messages = new String[] {DATA_MODIFICATION_ERROR};
+        final boolean[] throwables = new boolean[] {false};
+
+        try (LogbackTester logbackTester = new LogbackTester(BaseDataObject.class.getName())) {
+            final IBaseDataObject ibdo = new BaseDataObject();
+
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(DATA_MODIFICATION_BYTES));
+
+            final byte[] data0 = ibdo.data();
+            final byte[] data1 = ibdo.data();
+
+            Arrays.fill(data1, (byte) 0);
+
+            ibdo.checkAndResetArrayHashMap("TestPlace");
+
+            assertArrayEquals(DATA_MODIFICATION_BYTES, ibdo.data());
+            logbackTester.checkLogList(levels, messages, throwables);
+        }
+    }
+
+    @Test
+    void testChannelFactoryInTwoArrayOutNoSet2() throws IOException {
+        final Level[] levels = new Level[] {Level.WARN, Level.WARN};
+        final String[] messages = new String[] {DATA_MODIFICATION_ERROR, DATA_MODIFICATION_ERROR};
+        final boolean[] throwables = new boolean[] {false, false};
+
+        try (LogbackTester logbackTester = new LogbackTester(BaseDataObject.class.getName())) {
+            final IBaseDataObject ibdo = new BaseDataObject();
+
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(DATA_MODIFICATION_BYTES));
+
+            final byte[] data0 = ibdo.data();
+            final byte[] data1 = ibdo.data();
+
+            Arrays.fill(data0, (byte) 0);
+            Arrays.fill(data1, (byte) 0);
+
+            ibdo.checkAndResetArrayHashMap("TestPlace");
+
+            assertArrayEquals(DATA_MODIFICATION_BYTES, ibdo.data());
+            logbackTester.checkLogList(levels, messages, throwables);
+        }
+    }
+
+    @Test
+    void testChannelFactoryInArrayOutSetChannelFactory() throws IOException {
+        try (LogbackTester logbackTester = new LogbackTester(BaseDataObject.class.getName())) {
+            final IBaseDataObject ibdo = new BaseDataObject();
+
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(DATA_MODIFICATION_BYTES));
+
+            final byte[] data = ibdo.data();
+
+            Arrays.fill(data, (byte) 0);
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(data));
+
+            ibdo.checkAndResetArrayHashMap("TestPlace");
+
+            assertArrayEquals(new byte[DATA_MODIFICATION_BYTES.length], ibdo.data());
+            logbackTester.checkLogList(new Level[0], new String[0], new boolean[0]);
+        }
+    }
+
+    @Test
+    void testChannelFactoryInArrayOutSetArray() throws IOException {
+        try (LogbackTester logbackTester = new LogbackTester(BaseDataObject.class.getName())) {
+            final IBaseDataObject ibdo = new BaseDataObject();
+
+            ibdo.setChannelFactory(InMemoryChannelFactory.create(DATA_MODIFICATION_BYTES));
+
+            final byte[] data = ibdo.data();
+
+            Arrays.fill(data, (byte) 0);
+            ibdo.setData(data);
+
+            ibdo.checkAndResetArrayHashMap("TestPlace");
+
+            assertArrayEquals(new byte[DATA_MODIFICATION_BYTES.length], ibdo.data());
+            logbackTester.checkLogList(new Level[0], new String[0], new boolean[0]);
+        }
+    }
 }

src/test/java/emissary/test/core/junit5/LogbackTester.java

@@ -0,0 +1,55 @@
+package emissary.test.core.junit5;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import org.apache.commons.lang3.Validate;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class LogbackTester implements Closeable {
+    public final String name;
+    public final Logger logger;
+    public final ListAppender<ILoggingEvent> appender;
+
+    public LogbackTester(final String name) {
+        Validate.notNull(name, "Required: name != null");
+
+        this.name = name;
+        logger = (Logger) LoggerFactory.getLogger(name);
+        appender = new ListAppender<>();
+
+        appender.setContext(logger.getLoggerContext());
+        appender.start();
+        logger.addAppender(appender);
+        logger.setAdditive(false);
+    }
+
+    public void checkLogList(final Level[] levels, final String[] messages, final boolean[] throwables) {
+        Validate.notNull(levels, "Required: levels != null");
+        Validate.notNull(messages, "Required: messages != null");
+        Validate.notNull(throwables, "Required: throwables != null");
+        Validate.isTrue(levels.length == messages.length, "Required: levels.length == messages.length");
+        Validate.isTrue(levels.length == throwables.length, "Required: levels.length == throwables.length");
+
+        assertEquals(levels.length, appender.list.size(), "Expected lengths do not match number of log messages");
+
+        for (int i = 0; i < appender.list.size(); i++) {
+            final ILoggingEvent item = appender.list.get(i);
+
+            assertEquals(levels[i], item.getLevel(), "Levels not equal for element " + i);
+            assertEquals(messages[i], item.getFormattedMessage(), "Messages not equal for element " + i);
+            assertEquals(throwables[i], item.getThrowableProxy() != null, "Throwables not equal for elmeent " + i);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        logger.detachAndStopAllAppenders();
+    }
+}