nv2a: Handle NaN values to be similar to HW

xemu-project · Oct 5, 2024 · 2ab43f1 · 2ab43f1
1 parent cc001ce
commit 2ab43f1
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 20 deletions.
diff --git a/hw/xbox/nv2a/shaders.c b/hw/xbox/nv2a/shaders.c
@@ -855,6 +855,20 @@ GLSL_DEFINE(texMat3, GLSL_C_MAT4(NV_IGRAPH_XF_XFCTX_T3MAT))
                                i, i);
         }
     }
+    mstring_append(header,
+                  "/* Converts the input to vec4, pads with last component */\n"
+                  "vec4 _in(float v) { return vec4(v); }\n"
+                  "vec4 _in(vec2 v) { return v.xyyy; }\n"
+                  "vec4 _in(vec3 v) { return v.xyzz; }\n"
+                  "vec4 _in(vec4 v) { return v.xyzw; }\n"
+                  "#define FixNaN(src, mask) _FixNaN(_in(src)).mask\n"
+                  "vec4 _FixNaN(vec4 src)\n"
+                  "{\n"
+                  "  if (any(isnan(src))) {\n"
+                  "    return mix(src, max(vec4(0.0), sign(src)), isnan(src));\n"
+                  "  }\n"
+                  "  return src;\n"
+                  "}\n");
     mstring_append(header, "\n");
 
     MString *body = mstring_from_str("void main() {\n");
@@ -965,15 +979,15 @@ GLSL_DEFINE(texMat3, GLSL_C_MAT4(NV_IGRAPH_XF_XFCTX_T3MAT))
     /* Set outputs */
     const char *shade_model_mult = state->smooth_shading ? "vtx_inv_w" : "vtx_inv_w_flat";
     mstring_append_fmt(body, "\n"
-                      "  vtxD0 = clamp(oD0, 0.0, 1.0) * %s;\n"
-                      "  vtxD1 = clamp(oD1, 0.0, 1.0) * %s;\n"
-                      "  vtxB0 = clamp(oB0, 0.0, 1.0) * %s;\n"
-                      "  vtxB1 = clamp(oB1, 0.0, 1.0) * %s;\n"
-                      "  vtxFog = oFog.x * vtx_inv_w;\n"
-                      "  vtxT0 = oT0 * vtx_inv_w;\n"
-                      "  vtxT1 = oT1 * vtx_inv_w;\n"
-                      "  vtxT2 = oT2 * vtx_inv_w;\n"
-                      "  vtxT3 = oT3 * vtx_inv_w;\n"
+                      "  vtxD0 = clamp(FixNaN(oD0, xyzw), 0.0, 1.0) * %s;\n"
+                      "  vtxD1 = clamp(FixNaN(oD1, xyzw), 0.0, 1.0) * %s;\n"
+                      "  vtxB0 = clamp(FixNaN(oB0, xyzw), 0.0, 1.0) * %s;\n"
+                      "  vtxB1 = clamp(FixNaN(oB1, xyzw), 0.0, 1.0) * %s;\n"
+                      "  vtxFog = FixNaN(oFog, x) * vtx_inv_w;\n"
+                      "  vtxT0 = FixNaN(oT0, xyzw) * vtx_inv_w;\n"
+                      "  vtxT1 = FixNaN(oT1, xyzw) * vtx_inv_w;\n"
+                      "  vtxT2 = FixNaN(oT2, xyzw) * vtx_inv_w;\n"
+                      "  vtxT3 = FixNaN(oT3, xyzw) * vtx_inv_w;\n"
                       "  gl_Position = oPos;\n"
                       "  gl_PointSize = oPts.x;\n"
                       "  gl_ClipDistance[0] = oPos.z - oPos.w*clipRange.z;\n" // Near

diff --git a/hw/xbox/nv2a/vsh.c b/hw/xbox/nv2a/vsh.c
@@ -614,17 +614,6 @@ static const char* vsh_header =
      * https://www.opengl.org/registry/specs/NV/vertex_program1_1.txt
      */
     "\n"
-//QQQ #ifdef NICE_CODE
-    "/* Converts the input to vec4, pads with last component */\n"
-    "vec4 _in(float v) { return vec4(v); }\n"
-    "vec4 _in(vec2 v) { return v.xyyy; }\n"
-    "vec4 _in(vec3 v) { return v.xyzz; }\n"
-    "vec4 _in(vec4 v) { return v.xyzw; }\n"
-//#else
-//    "/* Make sure input is always a vec4 */\n"
-//   "#define _in(v) vec4(v)\n"
-//#endif
-    "\n"
     "#define INFINITY (1.0 / 0.0)\n"
     "\n"
     "#define MOV(dest, mask, src) dest.mask = _MOV(_in(src)).mask\n"