gamut_mapping: soft-clip in RGB after mapping

The reason this new curve works so well is because it incorporates a bit
of hard clipping in RGB space, which is functionally equivalent to
applying a sort of very soft intensity-dependent rolloff curve. Make
this clipping soft rather than hard, and also an explicit part of the
gamut mapping function - so it shows up on the graphs, and also to make
the function well-behaved.

This does distort hues very slightly, but hopefully in an aesthetically
pleasing way.

src/gamut_mapping.c

@@ -716,7 +716,15 @@ hueshift_done: ;
         struct ICh target = saturate(ich.h, dst);
         ich.C = softclip(ich.C, margin * source.C, target.C);
 
-        ipt = ich2ipt(ich);
+        // Soft-clip the resulting RGB color. This will generally distort
+        // hues slightly, but hopefully in an aesthetically pleasing way.
+        struct ICh saturated = { ich.I, target.C, ich.h };
+        struct RGB peak = ipt2rgb(ich2ipt(saturated), dst);
+        struct RGB rgb = ipt2rgb(ich2ipt(ich), dst);
+        rgb.R = fmaxf(softclip(rgb.R, peak.R, dst.max_rgb), dst.min_rgb);
+        rgb.G = fmaxf(softclip(rgb.G, peak.G, dst.max_rgb), dst.min_rgb);
+        rgb.B = fmaxf(softclip(rgb.B, peak.B, dst.max_rgb), dst.min_rgb);
+        ipt = rgb2ipt(rgb, dst);
     }
 }
 

src/tests/tone_mapping.c

@@ -152,7 +152,7 @@ int main()
 
         float hue_mapped = atan2f(c[2], c[1]);
         float hue_ref = atan2f(ref[2], ref[1]);
-        REQUIRE_FEQ(hue_mapped, hue_ref, 1e-3);
+        REQUIRE_FEQ(hue_mapped, hue_ref, 1e-2);
     }
 
     float *tmp = malloc(sizeof(float[LUT3D_SIZE][LUT3D_SIZE][LUT3D_SIZE][3]));