[obs] fix 24-bit import support with dmabuf

obs/CMakeLists.txt

@@ -27,7 +27,7 @@ add_compile_options(
   "$<$<CONFIG:DEBUG>:-O0;-g3;-ggdb>"
 )
 
-set(EXE_FLAGS "-Wl,--gc-sections")
+set(EXE_FLAGS "-Wl,--gc-sections -z noexecstack")
 set(CMAKE_C_STANDARD 11)
 
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)
@@ -47,6 +47,38 @@ else()
   set(OBS_PLUGIN_PREFIX ${CMAKE_INSTALL_DATADIR}/obs/obs-plugins)
 endif()
 
+include(MakeObject)
+function(build_effects header_dir)
+  file(GLOB headers "${header_dir}/*.h")
+  set(OBS_EFFECT_PROCESSED)
+  foreach(effect ${ARGN})
+    set(out_f "${CMAKE_CURRENT_BINARY_DIR}/${effect}")
+    add_custom_command(OUTPUT "${out_f}"
+      COMMAND "${AWK}" -f "${CMAKE_CURRENT_SOURCE_DIR}/glsl.include.awk"
+              "${CMAKE_CURRENT_SOURCE_DIR}/${effect}" > "${out_f}"
+      MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/${effect}"
+      DEPENDS ${headers}
+      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/effect"
+      COMMENT "Preprocessing effect ${effect}"
+      VERBATIM
+    )
+  endforeach()
+
+  #  set(CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+  make_object(
+    OBS_EFFECT
+    ${ARGN}
+  )
+
+  set(OBS_EFFECT_OBJS "${OBS_EFFECT_OBJS}" PARENT_SCOPE)
+  set(OBS_EFFECT_INCS "${OBS_EFFECT_INCS}" PARENT_SCOPE)
+endfunction()
+
+build_effects(
+  effect
+  effect/rgb24.effect
+)
+
 add_definitions(-D ATOMIC_LOCKING)
 
 add_custom_command(
@@ -59,6 +91,7 @@ add_custom_command(
 include_directories(
   ${PROJECT_SOURCE_DIR}/include
   ${CMAKE_BINARY_DIR}/include
+  ${OBS_EFFECT_INCS}
 )
 
 link_libraries(
@@ -76,7 +109,10 @@ set(SOURCES
 add_subdirectory("${PROJECT_TOP}/common"          "${CMAKE_BINARY_DIR}/common")
 add_subdirectory("${PROJECT_TOP}/repos/LGMP/lgmp" "${CMAKE_BINARY_DIR}/lgmp"  )
 
-add_library(looking-glass-obs SHARED ${SOURCES})
+add_library(looking-glass-obs SHARED
+  ${SOURCES}
+  ${OBS_EFFECT_OBJS}
+)
 target_link_libraries(looking-glass-obs
   ${EXE_FLAGS}
   lg_common

obs/effect/rgb24.effect

@@ -0,0 +1,53 @@
+uniform float4x4  ViewProj;
+uniform texture2d image;
+uniform float2    outputSize;
+uniform int       swap;
+
+struct VertData
+{
+  float4 pos : POSITION;
+  float2 uv  : TEXCOORD0;
+};
+
+VertData VSDefault(VertData vert_in)
+{
+  VertData vert_out;
+  vert_out.pos = mul(float4(vert_in.pos.xyz, 1.0), ViewProj);
+  vert_out.uv  = vert_in.uv;
+  return vert_out;
+}
+
+float4 PSColorFilter(VertData vert_in) : TARGET
+{
+  uvec2 outputPos = uvec2(vert_in.uv * outputSize);
+
+  uint fst     = outputPos.x * 3u / 4u;
+  vec4 color_0 = texelFetch(image, ivec2(fst, outputPos.y), 0);
+
+  uint snd     = (outputPos.x * 3u + 1u) / 4u;
+  vec4 color_1 = texelFetch(image, ivec2(snd, outputPos.y), 0);
+
+  uint trd     = (outputPos.x * 3u + 2u) / 4u;
+  vec4 color_2 = texelFetch(image, ivec2(trd, outputPos.y), 0);
+
+  vec4 result = vec4(
+    color_0.barg[outputPos.x % 4u],
+    color_1.gbar[outputPos.x % 4u],
+    color_2.rgba[outputPos.x % 4u],
+    1.0
+  );
+
+  if (swap == 1)
+    return result.rgba;
+  else
+    return result.bgra;
+}
+
+technique Draw
+{
+  pass
+  {
+    vertex_shader = VSDefault(vert_in);
+    pixel_shader  = PSColorFilter(vert_in);
+  }
+}