Add support for UIA providers on non-root MSAA objects.

Apparently, Chrome supports this. It wasn't previously possible for an ElementIdentifier to have both an MSAA and UIA provider. In the case of Chrome, IRawElementProviderFragment is supported, but we don't really want to use it for navigation. So we assume that whichever interface gave an id is the "preferred" one.
madewokherd · Jul 31, 2024 · 482510b · 482510b
1 parent 26127e0
commit 482510b
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 38 deletions.
diff --git a/xalia/Win32/AccessibleProvider.cs b/xalia/Win32/AccessibleProvider.cs
@@ -7,6 +7,7 @@
 using Xalia.Gudl;
 using Xalia.UiDom;
 using static Xalia.Interop.Win32;
+using IServiceProvider = Xalia.Interop.Win32.IServiceProvider;
 
 namespace Xalia.Win32
 {
@@ -853,6 +854,49 @@ private List<ElementIdentifier> GetChildrenAccChildBackground(int count)
             return result;
         }
 
+        public static bool UiaProviderFromIAccessibleBackground(IAccessible obj, out IRawElementProviderSimple uiaprov)
+        {
+            // Returns true if this is a bridged UIA element.
+            // May return false and a non-null provider if it's a native IAccessible with a UIA provider.
+            if (UiaProviderFromIAccessible(obj, 0, UIA_PFIA_UNWRAP_BRIDGE, out uiaprov) == 0)
+            {
+                return true;
+            }
+
+            IServiceProvider sp = obj as IServiceProvider;
+
+            if (!(sp is null))
+            {
+                try
+                {
+                    Guid sid = IID_IAccessibleEx;
+                    var raw_accex = sp.QueryService(ref sid, ref sid);
+                    if (raw_accex != IntPtr.Zero)
+                    {
+                        object obj_accex = Marshal.GetObjectForIUnknown(raw_accex);
+                        uiaprov = obj_accex as IRawElementProviderSimple;
+                    }
+                }
+                catch (InvalidOperationException)
+                {
+                }
+                catch (NotImplementedException)
+                {
+                }
+                catch (InvalidCastException)
+                {
+                }
+                catch (ArgumentException)
+                {
+                }
+                catch (COMException)
+                {
+                }
+            }
+
+            return false;
+        }
+
         public static ElementIdentifier ElementIdFromVariantBackground(object variant, IAccessible base_acc, IntPtr root_hwnd)
         {
             ElementIdentifier result = default;
@@ -897,9 +941,10 @@ public static ElementIdentifier ElementIdFromVariantBackground(object variant, I
                 }
             }
 
-            // TODO: Check for IAccessibleEx (UIA)
+            UiaProviderFromIAccessibleBackground(acc, out var uiaprov);
 
             result.acc = acc;
+            result.prov = uiaprov;
 
             IAccessible2 acc2 = QueryIAccessible2(acc);
 
@@ -917,6 +962,27 @@ public static ElementIdentifier ElementIdFromVariantBackground(object variant, I
                 return result;
             }
 
+            if (!(uiaprov is null)) {
+                var fragment = uiaprov as IRawElementProviderFragment;
+                if (!(fragment is null))
+                {
+                    var runtime_id = fragment.GetRuntimeId();
+
+                    if (!(runtime_id is null))
+                    {
+                        if (runtime_id[0] == UiaAppendRuntimeId)
+                        {
+                            result.runtime_id = new int[runtime_id.Length + 2];
+                            result.runtime_id[0] = 42;
+                            result.runtime_id[1] = (int)root_hwnd;
+                            Array.Copy(runtime_id, 0, result.runtime_id, 2, runtime_id.Length);
+                        }
+                        else
+                            result.runtime_id = runtime_id;
+                    }
+                }
+            }
+
             // Identify object by IUnknown pointer
             result.punk = Marshal.GetIUnknownForObject(acc);
             Marshal.Release(result.punk);

diff --git a/xalia/Win32/HwndProvider.cs b/xalia/Win32/HwndProvider.cs
@@ -222,43 +222,8 @@ private async Task DiscoverProviders()
                         int hr = ObjectFromLresult(lr, IID_IAccessible, IntPtr.Zero, out var obj);
                         Marshal.ThrowExceptionForHR(hr);
 
-                        if (UiaProviderFromIAccessible(obj, 0, UIA_PFIA_UNWRAP_BRIDGE, out var uiaprov) == 0)
-                        {
+                        if (AccessibleProvider.UiaProviderFromIAccessibleBackground(obj, out var uiaprov))
                             obj = null;
-                        }
-                        else
-                        {
-                            IServiceProvider sp = obj as IServiceProvider;
-
-                            if (!(sp is null))
-                            {
-                                try
-                                {
-                                    Guid sid = IID_IAccessibleEx;
-                                    var raw_accex = sp.QueryService(ref sid, ref sid);
-                                    if (raw_accex != IntPtr.Zero)
-                                    {
-                                        object obj_accex = Marshal.GetObjectForIUnknown(raw_accex);
-                                        uiaprov = obj_accex as IRawElementProviderSimple;
-                                    }
-                                }
-                                catch (InvalidOperationException)
-                                {
-                                }
-                                catch (NotImplementedException)
-                                {
-                                }
-                                catch (InvalidCastException)
-                                {
-                                }
-                                catch (ArgumentException)
-                                {
-                                }
-                                catch (COMException)
-                                {
-                                }
-                            }
-                        }
 
                         return (obj, uiaprov);
                     }, CommandThreadPriority.Query);

diff --git a/xalia/Win32/Win32Connection.cs b/xalia/Win32/Win32Connection.cs
@@ -292,7 +292,7 @@ internal UiDomElement CreateElement(ElementIdentifier id)
             if (root_hwnd is null)
                 throw new InvalidOperationException("hwnd element must be created before child element");
 
-            if (!(id.prov is null))
+            if (!(id.prov is null) && !(id.runtime_id is null))
             {
                 result.AddProvider(new UiaProvider(root_hwnd, result, id.prov));
             }
@@ -306,6 +306,11 @@ internal UiDomElement CreateElement(ElementIdentifier id)
                 result.AddProvider(new AccessibleProvider(root_hwnd, result, id.acc, id.child_id));
             }
 
+            if (!(id.prov is null) && (id.runtime_id is null))
+            {
+                result.AddProvider(new UiaProvider(root_hwnd, result, id.prov));
+            }
+
             elements_by_id.Add(name, result);
 
             return result;