external textures are no longer restricted to SAMPLER_EXTERNAL

OpenGLES allows this when GL_OES_EGL_image is present.

android/filament-android/src/main/cpp/Texture.cpp

@@ -138,6 +138,13 @@ Java_com_google_android_filament_Texture_nBuilderImportTexture(JNIEnv*, jclass,
     builder->import((intptr_t)id);
 }
 
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_google_android_filament_Texture_nBuilderExternal(JNIEnv*, jclass, jlong nativeBuilder) {
+    Texture::Builder *builder = (Texture::Builder *) nativeBuilder;
+    builder->external();
+}
+
 extern "C" JNIEXPORT jlong JNICALL
 Java_com_google_android_filament_Texture_nBuilderBuild(JNIEnv*, jclass,
         jlong nativeBuilder, jlong nativeEngine) {

android/filament-android/src/main/java/com/google/android/filament/Texture.java

@@ -800,6 +800,19 @@ public Builder importTexture(long id) {
             return this;
         }
 
+        /**
+         * Creates an external texture. The content must be set using setExternalImage().
+         * The sampler can be SAMPLER_EXTERNAL or SAMPLER_2D depending on the format. Generally
+         * YUV formats must use SAMPLER_EXTERNAL. This depends on the backend features and is not
+         * validated.
+         * @return This Builder, for chaining calls.
+         */
+        @NonNull
+        public Builder external() {
+            nBuilderExternal(mNativeBuilder);
+            return this;
+        }
+
         /**
          * Creates a new <code>Texture</code> instance.
          * @param engine The {@link Engine} to associate this <code>Texture</code> with.
@@ -1261,6 +1274,7 @@ void clearNativeObject() {
     private static native void nBuilderUsage(long nativeBuilder, int flags);
     private static native void nBuilderSwizzle(long nativeBuilder, int r, int g, int b, int a);
     private static native void nBuilderImportTexture(long nativeBuilder, long id);
+    private static native void nBuilderExternal(long nativeBuilder);
     private static native long nBuilderBuild(long nativeBuilder, long nativeEngine);
 
     private static native int nGetWidth(long nativeTexture, int level);

filament/backend/include/private/backend/DriverAPI.inc

@@ -214,6 +214,7 @@ DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureViewSwizzle,
         backend::TextureSwizzle, a)
 
 DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureExternalImage,
+        backend::SamplerType, target,
         backend::TextureFormat, format,
         uint32_t, width,
         uint32_t, height,

filament/backend/src/metal/MetalDriver.mm

@@ -482,7 +482,9 @@
     mContext->textures.insert(construct_handle<MetalTexture>(th, *mContext, src, r, g, b, a));
 }
 
-void MetalDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::TextureFormat format,
+void MetalDriver::createTextureExternalImageR(Handle<HwTexture> th,
+        backend::SamplerType target,
+        backend::TextureFormat format,
         uint32_t width, uint32_t height, backend::TextureUsage usage, void* image) {
     mContext->textures.insert(construct_handle<MetalTexture>(
             th, *mContext, format, width, height, usage, (CVPixelBufferRef)image));

filament/backend/src/opengl/GLDescriptorSet.cpp

@@ -173,6 +173,7 @@ void GLDescriptorSet::update(OpenGLContext& gl,
 
             arg.target = t ? t->gl.target : 0;
             arg.id = t ? t->gl.id : 0;
+            arg.external = t ? t->gl.external :  false;
             if constexpr (std::is_same_v<T, Sampler> ||
                           std::is_same_v<T, SamplerWithAnisotropyWorkaround>) {
                 if constexpr (std::is_same_v<T, SamplerWithAnisotropyWorkaround>) {
@@ -285,7 +286,7 @@ void GLDescriptorSet::bind(
             } else if constexpr (std::is_same_v<T, Sampler>) {
                 GLuint const unit = p.getTextureUnit(set, binding);
                 if (arg.target) {
-                    gl.bindTexture(unit, arg.target, arg.id);
+                    gl.bindTexture(unit, arg.target, arg.id, arg.external);
                     gl.bindSampler(unit, arg.sampler);
                     if (UTILS_UNLIKELY(arg.ref)) {
                         updateTextureView(gl, handleAllocator, unit, arg);
@@ -296,7 +297,7 @@ void GLDescriptorSet::bind(
             } else if constexpr (std::is_same_v<T, SamplerWithAnisotropyWorkaround>) {
                 GLuint const unit = p.getTextureUnit(set, binding);
                 if (arg.target) {
-                    gl.bindTexture(unit, arg.target, arg.id);
+                    gl.bindTexture(unit, arg.target, arg.id, arg.external);
                     gl.bindSampler(unit, arg.sampler);
                     if (UTILS_UNLIKELY(arg.ref)) {
                         updateTextureView(gl, handleAllocator, unit, arg);
@@ -314,7 +315,7 @@ void GLDescriptorSet::bind(
                 // in ES2 the sampler parameters need to be set on the texture itself
                 GLuint const unit = p.getTextureUnit(set, binding);
                 if (arg.target) {
-                    gl.bindTexture(unit, arg.target, arg.id);
+                    gl.bindTexture(unit, arg.target, arg.id, arg.external);
                     SamplerParams const params = arg.params;
                     glTexParameteri(arg.target, GL_TEXTURE_MIN_FILTER,
                             (GLint)GLUtils::getTextureFilter(params.filterMin));

filament/backend/src/opengl/GLDescriptorSet.h

@@ -111,7 +111,9 @@ struct GLDescriptorSet : public HwDescriptorSet {
 
     // A sampler descriptor
     struct Sampler {
-        GLenum target = 0;                      // 4
+        uint16_t target;                        // 2 (GLenum)
+        bool external = false;                  // 1
+        bool reserved = false;                  // 1
         GLuint id = 0;                          // 4
         GLuint sampler = 0;                     // 4
         Handle<GLTextureRef> ref;               // 4
@@ -126,7 +128,9 @@ struct GLDescriptorSet : public HwDescriptorSet {
     };
 
     struct SamplerWithAnisotropyWorkaround {
-        GLenum target = 0;                      // 4
+        uint16_t target;                        // 2 (GLenum)
+        bool external = false;                  // 1
+        bool reserved = false;                  // 1
         GLuint id = 0;                          // 4
         GLuint sampler = 0;                     // 4
         Handle<GLTextureRef> ref;               // 4
@@ -143,7 +147,9 @@ struct GLDescriptorSet : public HwDescriptorSet {
 
     // A sampler descriptor for ES2
     struct SamplerGLES2 {
-        GLenum target = 0;                      // 4
+        uint16_t target;                        // 2 (GLenum)
+        bool external = false;                  // 1
+        bool reserved = false;                  // 1
         GLuint id = 0;                          // 4
         SamplerParams params{};                 // 4
         float anisotropy = 1.0f;                // 4

filament/backend/src/opengl/GLTexture.h

@@ -35,7 +35,7 @@ struct GLTextureRef {
     GLTextureRef() = default;
     // view reference counter
     uint16_t count = 1;
-    // current per-view values of the texture (in GL we can only have a single View active at
+    // Current per-view values of the texture (in GL we can only have a single View active at
     // a time, and this tracks that state). It's used to avoid unnecessarily change state.
     int8_t baseLevel = 127;
     int8_t maxLevel = -1;
@@ -50,7 +50,7 @@ struct GLTextureRef {
 struct GLTexture : public HwTexture {
     using HwTexture::HwTexture;
     struct GL {
-        GL() noexcept : imported(false), sidecarSamples(1), reserved1(0) {}
+        GL() noexcept : imported(false), external(false), sidecarSamples(1), reserved1(0) {}
         GLuint id = 0;          // texture or renderbuffer id
         GLenum target = 0;
         GLenum internalFormat = 0;
@@ -62,7 +62,8 @@ struct GLTexture : public HwTexture {
         int8_t maxLevel = -1;
         uint8_t reserved0 = 0;
         bool imported           : 1;
-        uint8_t sidecarSamples  : 4;
+        bool external           : 1;
+        uint8_t sidecarSamples  : 3;
         uint8_t reserved1       : 3;
         std::array<TextureSwizzle, 4> swizzle{
                 TextureSwizzle::CHANNEL_0,

filament/backend/src/opengl/OpenGLContext.h

@@ -153,7 +153,7 @@ class OpenGLContext final : public TimerQueryFactoryInterface {
 
           void pixelStore(GLenum, GLint) noexcept;
     inline void activeTexture(GLuint unit) noexcept;
-    inline void bindTexture(GLuint unit, GLuint target, GLuint texId) noexcept;
+    inline void bindTexture(GLuint unit, GLuint target, GLuint texId, bool external) noexcept;
 
            void unbindTexture(GLenum target, GLuint id) noexcept;
            void unbindTextureUnit(GLuint unit) noexcept;
@@ -761,15 +761,17 @@ void OpenGLContext::bindBufferRange(GLenum target, GLuint index, GLuint buffer,
 #endif
 }
 
-void OpenGLContext::bindTexture(GLuint unit, GLuint target, GLuint texId) noexcept {
+void OpenGLContext::bindTexture(GLuint unit, GLuint target, GLuint texId, bool external) noexcept {
+    //  another texture is bound to the same unit with a different target,
+    //  unbind the texture from the current target
     update_state(state.textures.units[unit].target, target, [&]() {
         activeTexture(unit);
         glBindTexture(state.textures.units[unit].target, 0);
     });
     update_state(state.textures.units[unit].id, texId, [&]() {
         activeTexture(unit);
         glBindTexture(target, texId);
-    }, target == GL_TEXTURE_EXTERNAL_OES);
+    }, external);
 }
 
 void OpenGLContext::useProgram(GLuint program) noexcept {

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -402,7 +402,7 @@ void OpenGLDriver::setPushConstant(backend::ShaderStage stage, uint8_t index,
 
 void OpenGLDriver::bindTexture(GLuint unit, GLTexture const* t) noexcept {
     assert_invariant(t != nullptr);
-    mContext.bindTexture(unit, t->gl.target, t->gl.id);
+    mContext.bindTexture(unit, t->gl.target, t->gl.id, t->gl.external);
 }
 
 bool OpenGLDriver::useProgram(OpenGLProgram* p) noexcept {
@@ -1052,14 +1052,36 @@ void OpenGLDriver::createTextureViewSwizzleR(Handle<HwTexture> th, Handle<HwText
     CHECK_GL_ERROR(utils::slog.e)
 }
 
-void OpenGLDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::TextureFormat format,
-        uint32_t width, uint32_t height, backend::TextureUsage usage, void* image) {
+void OpenGLDriver::createTextureExternalImageR(Handle<HwTexture> th, SamplerType target,
+    TextureFormat format, uint32_t width, uint32_t height, TextureUsage usage, void* image) {
     DEBUG_MARKER()
-    createTextureR(th, SamplerType::SAMPLER_EXTERNAL, 1, format, 1, width, height, 1, usage);
 
-    GLTexture* t = handle_cast<GLTexture*>(th);
+    usage |= TextureUsage::SAMPLEABLE;
+    usage &= ~TextureUsage::UPLOADABLE;
+
+    auto& gl = mContext;
+    GLenum internalFormat = getInternalFormat(format);
+    if (UTILS_UNLIKELY(gl.isES2())) {
+        // on ES2, format and internal format must match
+        // FIXME: handle compressed texture format
+        internalFormat = textureFormatToFormatAndType(format).first;
+    }
+    assert_invariant(internalFormat);
+
+    GLTexture* const t = construct<GLTexture>(th, target, 1, 1, width, height, 1, format, usage);
     assert_invariant(t);
-    assert_invariant(t->target == SamplerType::SAMPLER_EXTERNAL);
+
+    t->externalTexture = mPlatform.createExternalImageTexture();
+    if (t->externalTexture) {
+        t->gl.target = t->externalTexture->target;
+        t->gl.id = t->externalTexture->id;
+        // internalFormat actually depends on the external image, but it doesn't matter
+        // because it's not used anywhere for anything important.
+        t->gl.internalFormat = internalFormat;
+        t->gl.baseLevel = 0;
+        t->gl.maxLevel = 0;
+        t->gl.external = true; // forces bindTexture() call (they're never cached)
+    }
 
     bindTexture(OpenGLContext::DUMMY_TEXTURE_BINDING, t);
     if (mPlatform.setExternalImage(image, t->externalTexture)) {
@@ -1092,6 +1114,7 @@ void OpenGLDriver::importTextureR(Handle<HwTexture> th, intptr_t id,
     switch (target) {
         case SamplerType::SAMPLER_EXTERNAL:
             t->gl.target = GL_TEXTURE_EXTERNAL_OES;
+            t->gl.external = true; // forces bindTexture() call (they're never cached)
             break;
         case SamplerType::SAMPLER_2D:
             t->gl.target = GL_TEXTURE_2D;
@@ -3503,7 +3526,7 @@ void OpenGLDriver::endFrame(UTILS_UNUSED uint32_t frameId) {
     auto& gl = mContext;
     gl.bindVertexArray(nullptr);
     for (int unit = OpenGLContext::DUMMY_TEXTURE_BINDING; unit >= 0; unit--) {
-        gl.bindTexture(unit, GL_TEXTURE_2D, 0);
+        gl.bindTexture(unit, GL_TEXTURE_2D, 0, false);
     }
     gl.disable(GL_CULL_FACE);
     gl.depthFunc(GL_LESS);

filament/backend/src/opengl/platforms/PlatformEGL.cpp

@@ -715,14 +715,16 @@ void PlatformEGL::destroyExternalImage(ExternalTexture* texture) noexcept {
 
 bool PlatformEGL::setExternalImage(void* externalImage,
         UTILS_UNUSED_IN_RELEASE ExternalTexture* texture) noexcept {
-    if (UTILS_LIKELY(ext.gl.OES_EGL_image_external_essl3)) {
-        assert_invariant(texture->target == GL_TEXTURE_EXTERNAL_OES);
+
+    // OES_EGL_image_external_essl3 must be present if the target is TEXTURE_EXTERNAL_OES
+    // GL_OES_EGL_image must be present if TEXTURE_2D is used
+
+#if defined(GL_OES_EGL_image) || defined(GL_OES_EGL_image_external_essl3)
         // the texture is guaranteed to be bound here.
-#ifdef GL_OES_EGL_image
-        glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES,
+        glEGLImageTargetTexture2DOES(texture->target,
                 static_cast<GLeglImageOES>(externalImage));
 #endif
-    }
+
     return true;
 }
 

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -543,7 +543,8 @@ void VulkanDriver::createTextureViewSwizzleR(Handle<HwTexture> th, Handle<HwText
    texture.inc();
 }
 
-void VulkanDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::TextureFormat format,
+void VulkanDriver::createTextureExternalImageR(Handle<HwTexture> th,
+        backend::SamplerType target,  backend::TextureFormat format,
         uint32_t width, uint32_t height, backend::TextureUsage usage, void* externalImage) {
     FVK_SYSTRACE_SCOPE();
 

filament/backend/test/test_RenderExternalImage.cpp

@@ -227,7 +227,8 @@ TEST_F(BackendTest, RenderExternalImage) {
 
     api.setupExternalImage(pixBuffer);
     backend::Handle<HwTexture> texture =
-            api.createTextureExternalImage(TextureFormat::RGBA8, 1024, 1024, usage, pixBuffer);
+            api.createTextureExternalImage(SamplerType::SAMPLER_EXTERNAL,
+                    TextureFormat::RGBA8, 1024, 1024, usage, pixBuffer);
 
     // We're now free to release the buffer.
     CVBufferRelease(pixBuffer);

filament/include/filament/Texture.h

@@ -261,6 +261,18 @@ class UTILS_PUBLIC Texture : public FilamentAPI {
          */
         Builder& import(intptr_t id) noexcept;
 
+        /**
+         * Creates an external texture. The content must be set using setExternalImage().
+         * The sampler can be SAMPLER_EXTERNAL or SAMPLER_2D depending on the format. Generally
+         * YUV formats must use SAMPLER_EXTERNAL. This depends on the backend features and is not
+         * validated.
+         *
+         * If the Sampler is set to SAMPLER_EXTERNAL, external() is implied.
+         *
+         * @return
+         */
+        Builder& external() noexcept;
+
     private:
         friend class FTexture;
     };
@@ -395,7 +407,7 @@ class UTILS_PUBLIC Texture : public FilamentAPI {
 
 
     /**
-     * Specify the external image to associate with this Texture. Typically the external
+     * Specify the external image to associate with this Texture. Typically, the external
      * image is OS specific, and can be a video or camera frame.
      * There are many restrictions when using an external image as a texture, such as:
      *   - only the level of detail (lod) 0 can be specified
@@ -420,7 +432,7 @@ class UTILS_PUBLIC Texture : public FilamentAPI {
     void setExternalImage(Engine& engine, void* UTILS_NONNULL image) noexcept;
 
     /**
-     * Specify the external image and plane to associate with this Texture. Typically the external
+     * Specify the external image and plane to associate with this Texture. Typically, the external
      * image is OS specific, and can be a video or camera frame. When using this method, the
      * external image must be a planar type (such as a YUV camera frame). The plane parameter
      * selects which image plane is bound to this texture.