[host] dxgi: correct 24-bit packed output height calculation bug

gnif · Jan 27, 2024 · 84dd68d · 84dd68d
1 parent 738a04d
commit 84dd68d
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/host/platform/Windows/capture/DXGI/src/pp/rgb24.c b/host/platform/Windows/capture/DXGI/src/pp/rgb24.c
@@ -78,11 +78,13 @@ static bool rgb24_configure(void * opaque,
 
   if (!this.pshader)
   {
+    const unsigned packedPitch = ALIGN_TO(*cols * 3, 4);
+
     /* we must align to 64 byte boundaries to avoid breaking dmabuf import */
-    this.width  = ALIGN_TO((*cols * 3 + 3) / 4, 64);
+    this.width  = ALIGN_TO(packedPitch / 4, 64);
 
     /* adjust for the aligned width */
-    this.height = (((*rows * *cols) * 3 + 3) / 4) / this.width;
+    this.height = (*cols * *rows) / (packedPitch / 3);
 
     char sOutputWidth[6], sOutputHeight[6];
     snprintf(sOutputWidth , sizeof(sOutputWidth) , "%d", this.width );