From 482510b7c4b7cf8b31e041146dcf3b8dadf6b66b Mon Sep 17 00:00:00 2001 From: Esme Povirk Date: Wed, 31 Jul 2024 19:33:29 +0000 Subject: [PATCH] Add support for UIA providers on non-root MSAA objects. Apparently, Chrome supports this. It wasn't previously possible for an ElementIdentifier to have both an MSAA and UIA provider. In the case of Chrome, IRawElementProviderFragment is supported, but we don't really want to use it for navigation. So we assume that whichever interface gave an id is the "preferred" one. --- xalia/Win32/AccessibleProvider.cs | 68 ++++++++++++++++++++++++++++++- xalia/Win32/HwndProvider.cs | 37 +---------------- xalia/Win32/Win32Connection.cs | 7 +++- 3 files changed, 74 insertions(+), 38 deletions(-) diff --git a/xalia/Win32/AccessibleProvider.cs b/xalia/Win32/AccessibleProvider.cs index dc71d3e..d5c6d4b 100644 --- a/xalia/Win32/AccessibleProvider.cs +++ b/xalia/Win32/AccessibleProvider.cs @@ -7,6 +7,7 @@ using Xalia.Gudl; using Xalia.UiDom; using static Xalia.Interop.Win32; +using IServiceProvider = Xalia.Interop.Win32.IServiceProvider; namespace Xalia.Win32 { @@ -853,6 +854,49 @@ private List GetChildrenAccChildBackground(int count) return result; } + public static bool UiaProviderFromIAccessibleBackground(IAccessible obj, out IRawElementProviderSimple uiaprov) + { + // Returns true if this is a bridged UIA element. + // May return false and a non-null provider if it's a native IAccessible with a UIA provider. + if (UiaProviderFromIAccessible(obj, 0, UIA_PFIA_UNWRAP_BRIDGE, out uiaprov) == 0) + { + return true; + } + + IServiceProvider sp = obj as IServiceProvider; + + if (!(sp is null)) + { + try + { + Guid sid = IID_IAccessibleEx; + var raw_accex = sp.QueryService(ref sid, ref sid); + if (raw_accex != IntPtr.Zero) + { + object obj_accex = Marshal.GetObjectForIUnknown(raw_accex); + uiaprov = obj_accex as IRawElementProviderSimple; + } + } + catch (InvalidOperationException) + { + } + catch (NotImplementedException) + { + } + catch (InvalidCastException) + { + } + catch (ArgumentException) + { + } + catch (COMException) + { + } + } + + return false; + } + public static ElementIdentifier ElementIdFromVariantBackground(object variant, IAccessible base_acc, IntPtr root_hwnd) { ElementIdentifier result = default; @@ -897,9 +941,10 @@ public static ElementIdentifier ElementIdFromVariantBackground(object variant, I } } - // TODO: Check for IAccessibleEx (UIA) + UiaProviderFromIAccessibleBackground(acc, out var uiaprov); result.acc = acc; + result.prov = uiaprov; IAccessible2 acc2 = QueryIAccessible2(acc); @@ -917,6 +962,27 @@ public static ElementIdentifier ElementIdFromVariantBackground(object variant, I return result; } + if (!(uiaprov is null)) { + var fragment = uiaprov as IRawElementProviderFragment; + if (!(fragment is null)) + { + var runtime_id = fragment.GetRuntimeId(); + + if (!(runtime_id is null)) + { + if (runtime_id[0] == UiaAppendRuntimeId) + { + result.runtime_id = new int[runtime_id.Length + 2]; + result.runtime_id[0] = 42; + result.runtime_id[1] = (int)root_hwnd; + Array.Copy(runtime_id, 0, result.runtime_id, 2, runtime_id.Length); + } + else + result.runtime_id = runtime_id; + } + } + } + // Identify object by IUnknown pointer result.punk = Marshal.GetIUnknownForObject(acc); Marshal.Release(result.punk); diff --git a/xalia/Win32/HwndProvider.cs b/xalia/Win32/HwndProvider.cs index 2cf3b04..ac070ff 100644 --- a/xalia/Win32/HwndProvider.cs +++ b/xalia/Win32/HwndProvider.cs @@ -222,43 +222,8 @@ private async Task DiscoverProviders() int hr = ObjectFromLresult(lr, IID_IAccessible, IntPtr.Zero, out var obj); Marshal.ThrowExceptionForHR(hr); - if (UiaProviderFromIAccessible(obj, 0, UIA_PFIA_UNWRAP_BRIDGE, out var uiaprov) == 0) - { + if (AccessibleProvider.UiaProviderFromIAccessibleBackground(obj, out var uiaprov)) obj = null; - } - else - { - IServiceProvider sp = obj as IServiceProvider; - - if (!(sp is null)) - { - try - { - Guid sid = IID_IAccessibleEx; - var raw_accex = sp.QueryService(ref sid, ref sid); - if (raw_accex != IntPtr.Zero) - { - object obj_accex = Marshal.GetObjectForIUnknown(raw_accex); - uiaprov = obj_accex as IRawElementProviderSimple; - } - } - catch (InvalidOperationException) - { - } - catch (NotImplementedException) - { - } - catch (InvalidCastException) - { - } - catch (ArgumentException) - { - } - catch (COMException) - { - } - } - } return (obj, uiaprov); }, CommandThreadPriority.Query); diff --git a/xalia/Win32/Win32Connection.cs b/xalia/Win32/Win32Connection.cs index a44fd9a..f344a97 100644 --- a/xalia/Win32/Win32Connection.cs +++ b/xalia/Win32/Win32Connection.cs @@ -292,7 +292,7 @@ internal UiDomElement CreateElement(ElementIdentifier id) if (root_hwnd is null) throw new InvalidOperationException("hwnd element must be created before child element"); - if (!(id.prov is null)) + if (!(id.prov is null) && !(id.runtime_id is null)) { result.AddProvider(new UiaProvider(root_hwnd, result, id.prov)); } @@ -306,6 +306,11 @@ internal UiDomElement CreateElement(ElementIdentifier id) result.AddProvider(new AccessibleProvider(root_hwnd, result, id.acc, id.child_id)); } + if (!(id.prov is null) && (id.runtime_id is null)) + { + result.AddProvider(new UiaProvider(root_hwnd, result, id.prov)); + } + elements_by_id.Add(name, result); return result;