Merge pull request #780 from zeux/vcodec-trace

vertexcodec: Reintroduce tracing and add roundtrip fuzzing

src/vertexcodec.cpp

@@ -90,6 +90,14 @@
 #include <wasm_simd128.h>
 #endif
 
+#ifndef TRACE
+#define TRACE 0
+#endif
+
+#if TRACE
+#include <stdio.h>
+#endif
+
 #ifdef SIMD_WASM
 #define wasmx_splat_v32x4(v, i) wasm_i32x4_shuffle(v, v, i, i, i, i)
 #define wasmx_unpacklo_v8x16(a, b) wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)
@@ -135,6 +143,19 @@ inline unsigned char unzigzag8(unsigned char v)
 	return -(v & 1) ^ (v >> 1);
 }
 
+#if TRACE
+struct Stats
+{
+	size_t size;
+	size_t header;  // bytes for header
+	size_t bitg[4]; // bytes for bit groups
+	size_t bitc[8]; // bit consistency: how many bits are shared between all bytes in a group
+};
+
+static Stats* bytestats = NULL;
+static Stats vertexstats[256];
+#endif
+
 static bool encodeBytesGroupZero(const unsigned char* buffer)
 {
 	for (size_t i = 0; i < kByteGroupSize; ++i)
@@ -256,8 +277,16 @@ static unsigned char* encodeBytes(unsigned char* data, unsigned char* data_end,
 
 		assert(data + best_size == next);
 		data = next;
+
+#if TRACE
+		bytestats->bitg[bitslog2] += best_size;
+#endif
 	}
 
+#if TRACE
+	bytestats->header += header_size;
+#endif
+
 	return data;
 }
 
@@ -286,9 +315,31 @@ static unsigned char* encodeVertexBlock(unsigned char* data, unsigned char* data
 			vertex_offset += vertex_size;
 		}
 
+#if TRACE
+		const unsigned char* olddata = data;
+		bytestats = &vertexstats[k];
+
+		for (size_t ig = 0; ig < vertex_count; ig += kByteGroupSize)
+		{
+			unsigned char last = (ig == 0) ? last_vertex[k] : vertex_data[vertex_size * (ig - 1) + k];
+			unsigned char delta = 0xff;
+
+			for (size_t i = ig; i < ig + kByteGroupSize && i < vertex_count; ++i)
+				delta &= ~(vertex_data[vertex_size * i + k] ^ last);
+
+			for (int j = 0; j < 8; ++j)
+				bytestats->bitc[j] += (vertex_count - ig < kByteGroupSize ? vertex_count - ig : kByteGroupSize) * ((delta >> j) & 1);
+		}
+#endif
+
 		data = encodeBytes(data, data_end, buffer, (vertex_count + kByteGroupSize - 1) & ~(kByteGroupSize - 1));
 		if (!data)
 			return NULL;
+
+#if TRACE
+		bytestats = NULL;
+		vertexstats[k].size += data - olddata;
+#endif
 	}
 
 	memcpy(last_vertex, &vertex_data[vertex_size * (vertex_count - 1)], vertex_size);
@@ -1096,6 +1147,10 @@ size_t meshopt_encodeVertexBuffer(unsigned char* buffer, size_t buffer_size, con
 	assert(vertex_size > 0 && vertex_size <= 256);
 	assert(vertex_size % 4 == 0);
 
+#if TRACE
+	memset(vertexstats, 0, sizeof(vertexstats));
+#endif
+
 	const unsigned char* vertex_data = static_cast<const unsigned char*>(vertices);
 
 	unsigned char* data = buffer;
@@ -1148,6 +1203,30 @@ size_t meshopt_encodeVertexBuffer(unsigned char* buffer, size_t buffer_size, con
 	assert(data >= buffer + tail_size);
 	assert(data <= buffer + buffer_size);
 
+#if TRACE
+	size_t total_size = data - buffer;
+
+	for (size_t k = 0; k < vertex_size; ++k)
+	{
+		const Stats& vsk = vertexstats[k];
+
+		printf("%2d: %7d bytes [%4.1f%%] %.1f bpv", int(k), int(vsk.size), double(vsk.size) / double(total_size) * 100, double(vsk.size) / double(vertex_count) * 8);
+
+		size_t total_k = vsk.header + vsk.bitg[0] + vsk.bitg[1] + vsk.bitg[2] + vsk.bitg[3];
+
+		printf(" |\thdr [%5.1f%%] bitg 1-3 [%4.1f%% %4.1f%% %4.1f%%]",
+		    double(vsk.header) / double(total_k) * 100, double(vsk.bitg[1]) / double(total_k) * 100,
+		    double(vsk.bitg[2]) / double(total_k) * 100, double(vsk.bitg[3]) / double(total_k) * 100);
+
+		printf(" |\tbitc [%3.0f%% %3.0f%% %3.0f%% %3.0f%% %3.0f%% %3.0f%% %3.0f%% %3.0f%%]",
+		    double(vsk.bitc[0]) / double(vertex_count) * 100, double(vsk.bitc[1]) / double(vertex_count) * 100,
+		    double(vsk.bitc[2]) / double(vertex_count) * 100, double(vsk.bitc[3]) / double(vertex_count) * 100,
+		    double(vsk.bitc[4]) / double(vertex_count) * 100, double(vsk.bitc[5]) / double(vertex_count) * 100,
+		    double(vsk.bitc[6]) / double(vertex_count) * 100, double(vsk.bitc[7]) / double(vertex_count) * 100);
+		printf("\n");
+	}
+#endif
+
 	return data - buffer;
 }
 

tools/codecfuzz.cpp

@@ -2,6 +2,7 @@
 
 #include <stdint.h>
 #include <stdlib.h>
+#include <string.h>
 
 void fuzzDecoder(const uint8_t* data, size_t size, size_t stride, int (*decode)(void*, size_t, size_t, const unsigned char*, size_t))
 {
@@ -16,9 +17,25 @@ void fuzzDecoder(const uint8_t* data, size_t size, size_t stride, int (*decode)(
 	free(destination);
 }
 
-namespace meshopt
+void fuzzRoundtrip(const uint8_t* data, size_t size, size_t stride)
 {
-extern unsigned int cpuid;
+	size_t count = size / stride;
+
+	size_t bound = meshopt_encodeVertexBufferBound(count, stride);
+	void* encoded = malloc(bound);
+	void* decoded = malloc(count * stride);
+	assert(encoded && decoded);
+
+	size_t res = meshopt_encodeVertexBuffer(static_cast<unsigned char*>(encoded), bound, data, count, stride);
+	assert(res <= bound);
+
+	int rc = meshopt_decodeVertexBuffer(decoded, count, stride, static_cast<unsigned char*>(encoded), res);
+	assert(rc == 0);
+
+	assert(memcmp(data, decoded, count * stride) == 0);
+
+	free(decoded);
+	free(encoded);
 }
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
@@ -38,5 +55,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 	fuzzDecoder(data, size, 24, meshopt_decodeVertexBuffer);
 	fuzzDecoder(data, size, 32, meshopt_decodeVertexBuffer);
 
+	// encodeVertexBuffer/decodeVertexBuffer should roundtrip for any stride, check a few with different alignment mod 16
+	fuzzRoundtrip(data, size, 4);
+	fuzzRoundtrip(data, size, 16);
+	fuzzRoundtrip(data, size, 24);
+	fuzzRoundtrip(data, size, 32);
+
 	return 0;
 }