From 9533dfe05098f4a90ef198d52c37a74815496fde Mon Sep 17 00:00:00 2001 From: Felipe Erias Date: Fri, 29 Dec 2023 14:34:15 +0800 Subject: [PATCH] Open immersive experiences directly This PR implements support for opening immersive experiences directly as soon as the application is launched. We do this by adding a built-in extension which will activate a particular element in the page. We need to set the preference dom.vr.require-gesture to false so this script can launch immersive WebXR experiences. Media autoplay is also allowed in this mode. The information required to identify the element to activate is passed as parameters to the Intent: - open_in_immersive - open_in_immersive_parent_xpath - open_in_immersive_element_xpath - a target URL to open --- .../com/igalia/wolvic/VRBrowserActivity.java | 31 +++- .../wolvic/browser/PermissionDelegate.java | 3 +- .../ui/widgets/WidgetManagerDelegate.java | 1 + .../com/igalia/wolvic/ui/widgets/Windows.java | 37 ++++- app/src/main/AndroidManifest.xml | 2 +- .../extensions/wolvic_autowebxr/content.js | 152 ++++++++++++++++++ .../extensions/wolvic_autowebxr/manifest.json | 23 +++ app/src/main/res/raw/fxr_config.yaml | 4 +- 8 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 app/src/main/assets/extensions/wolvic_autowebxr/content.js create mode 100644 app/src/main/assets/extensions/wolvic_autowebxr/manifest.json diff --git a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java index 37e993a4f9e..9477456de31 100644 --- a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java +++ b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java @@ -128,6 +128,12 @@ public class VRBrowserActivity extends PlatformActivity implements WidgetManager public static final String EXTRA_KIOSK = "kiosk"; private static final long BATTERY_UPDATE_INTERVAL = 60 * 1_000_000_000L; // 60 seconds + private boolean mOpenInImmersive = false; + public static final String EXTRA_OPEN_IN_IMMERSIVE = "open_in_immersive"; + // Element where a click would be simulated to launch the WebXR experience. + public static final String EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH = "open_in_immersive_parent_xpath"; + public static final String EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH = "open_in_immersive_element_xpath"; + private BroadcastReceiver mCrashReceiver = new BroadcastReceiver() { @Override public void onReceive(Context context, Intent intent) { @@ -248,6 +254,8 @@ public void run() { private PlatformActivityPlugin mPlatformPlugin; private int mLastMotionEventWidgetHandle; private boolean mIsEyeTrackingSupported; + private String mImmersiveParentElementXPath; + private String mImmersiveTargetElementXPath; private boolean callOnAudioManager(Consumer fn) { if (mAudioManager == null) { @@ -886,6 +894,14 @@ void loadFromIntent(final Intent intent) { } openInKioskMode = extras.getBoolean(EXTRA_KIOSK, false); + + if (extras.getBoolean(EXTRA_OPEN_IN_IMMERSIVE)) { + mImmersiveParentElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH); + mImmersiveTargetElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH); + + // Open in immersive requires specific information to be present + mOpenInImmersive = targetUri != null && mImmersiveTargetElementXPath != null; + } } // If there is a target URI we open it @@ -897,6 +913,8 @@ void loadFromIntent(final Intent intent) { if (openInKioskMode) { // FIXME this might not work as expected if the app was already running mWindows.openInKioskMode(targetUri.toString()); + } if (mOpenInImmersive) { + mWindows.openInImmersiveMode(targetUri, mImmersiveParentElementXPath, mImmersiveTargetElementXPath); } else { if (openInWindow) { location = Windows.OPEN_IN_NEW_WINDOW; @@ -1338,6 +1356,13 @@ void onExitWebXR(long aCallback) { return; } mIsPresentingImmersive = false; + TelemetryService.stopImmersive(); + + if (mOpenInImmersive) { + Log.d(LOGTAG, "Started in immersive mode: exiting WebXR will finish the app"); + finish(); + } + runOnUiThread(() -> { mWindows.exitImmersiveMode(); for (WebXRListener listener: mWebXRListeners) { @@ -1348,7 +1373,6 @@ void onExitWebXR(long aCallback) { // Show the window in front of you when you exit immersive mode. recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL); - TelemetryService.stopImmersive(); Handler handler = new Handler(Looper.getMainLooper()); handler.postDelayed(() -> { if (!mWindows.isPaused()) { @@ -1361,6 +1385,7 @@ void onExitWebXR(long aCallback) { } }, 20); } + @Keep @SuppressWarnings("unused") void onDismissWebXRInterstitial() { @@ -1887,6 +1912,10 @@ public boolean isWebXRPresenting() { return mIsPresentingImmersive; } + @Override public boolean isOpenInImmersive() { + return mOpenInImmersive; + } + @Override public void pushBackHandler(@NonNull Runnable aRunnable) { mBackHandlers.addLast(aRunnable); diff --git a/app/src/common/shared/com/igalia/wolvic/browser/PermissionDelegate.java b/app/src/common/shared/com/igalia/wolvic/browser/PermissionDelegate.java index 0609698b5b1..395b3ccb629 100644 --- a/app/src/common/shared/com/igalia/wolvic/browser/PermissionDelegate.java +++ b/app/src/common/shared/com/igalia/wolvic/browser/PermissionDelegate.java @@ -221,7 +221,8 @@ public WResult onContentPermissionRequest(WSession aSession, ContentPer // https://hacks.mozilla.org/2019/02/firefox-66-to-block-automatically-playing-audible-video-and-audio/ return WResult.fromValue(ContentPermission.VALUE_ALLOW); } else if(perm.permission == PERMISSION_AUTOPLAY_AUDIBLE) { - if (SettingsStore.getInstance(mContext).isAutoplayEnabled()) { + // allow autoplay when we start Wolvic in immersive mode automatically + if (SettingsStore.getInstance(mContext).isAutoplayEnabled() || mWidgetManager.isOpenInImmersive()) { return WResult.fromValue(ContentPermission.VALUE_ALLOW); } else { return WResult.fromValue(ContentPermission.VALUE_DENY); diff --git a/app/src/common/shared/com/igalia/wolvic/ui/widgets/WidgetManagerDelegate.java b/app/src/common/shared/com/igalia/wolvic/ui/widgets/WidgetManagerDelegate.java index de944321c0f..7e565df98be 100644 --- a/app/src/common/shared/com/igalia/wolvic/ui/widgets/WidgetManagerDelegate.java +++ b/app/src/common/shared/com/igalia/wolvic/ui/widgets/WidgetManagerDelegate.java @@ -121,6 +121,7 @@ enum OriginatorType {WEBSITE, APPLICATION} boolean isWebXRIntersitialHidden(); boolean isWebXRPresenting(); boolean isPermissionGranted(@NonNull String permission); + boolean isOpenInImmersive(); void requestPermission(String originator, @NonNull String permission, OriginatorType originatorType, WSession.PermissionDelegate.Callback aCallback); boolean canOpenNewWindow(); void openNewWindow(String uri); diff --git a/app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java b/app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java index f48ee4df194..7f609bd8949 100644 --- a/app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java +++ b/app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java @@ -4,6 +4,7 @@ import android.content.Context; import android.content.SharedPreferences; +import android.net.Uri; import android.util.Log; import androidx.annotation.IntDef; @@ -75,7 +76,6 @@ public class Windows implements TrayListener, TopBarWidget.Delegate, TitleBarWid public static final int OPEN_IN_BACKGROUND = 1; public static final int OPEN_IN_NEW_WINDOW = 2; - private static final String WINDOWS_SAVE_FILENAME = "windows_state.json"; private static final int TAB_ADDED_NOTIFICATION_ID = 0; @@ -83,6 +83,12 @@ public class Windows implements TrayListener, TopBarWidget.Delegate, TitleBarWid private static final int BOOKMARK_ADDED_NOTIFICATION_ID = 2; private static final int WEB_APP_ADDED_NOTIFICATION_ID = 3; + // start Wolvic in immersive mode automatically + private static final String PARENT_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-parentElementXPath"; + private static final String TARGET_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-targetElementXPath"; + private static final String IMMERSIVE_EXTENSION_ID = "wolvic-autowebxr@igalia.com"; + private static final String IMMERSIVE_EXTENSION_URL = "resource://android/assets/extensions/wolvic_autowebxr/"; + class WindowState { WindowPlacement placement; int textureWidth; @@ -1475,6 +1481,35 @@ public void openInKioskMode(@NonNull String aUri) { mFocusedWindow.setKioskMode(true); } + public void openInImmersiveMode(Uri targetUri, String immersiveParentElementXPath, String immersiveTargetElementXPath) { + Uri.Builder uriBuilder = targetUri.buildUpon(); + if (!StringUtils.isEmpty(immersiveParentElementXPath)) { + uriBuilder.appendQueryParameter(PARENT_ELEMENT_XPATH_PARAMETER, immersiveParentElementXPath); + } + if (!StringUtils.isEmpty(immersiveTargetElementXPath)) { + uriBuilder.appendQueryParameter(TARGET_ELEMENT_XPATH_PARAMETER, immersiveTargetElementXPath); + } + Uri extendedUri = uriBuilder.build(); + + Session session = SessionStore.get().createSuspendedSession(extendedUri.toString(), true); + + mFocusedWindow.setKioskMode(true); + + SessionStore.get().getWebExtensionRuntime().installBuiltInWebExtension( + IMMERSIVE_EXTENSION_ID, + IMMERSIVE_EXTENSION_URL, + webExtension -> { + setFirstPaint(mFocusedWindow, session); + mFocusedWindow.setSession(session, WindowWidget.DEACTIVATE_CURRENT_SESSION); + return null; + }, + (throwable) -> { + Log.e(LOGTAG, "Error installing the " + IMMERSIVE_EXTENSION_ID + " from " + IMMERSIVE_EXTENSION_URL + " Web Extension: " + throwable.getLocalizedMessage()); + return null; + } + ); + } + public void addTab(@NonNull WindowWidget targetWindow, @Nullable String aUri) { Session session = SessionStore.get().createSuspendedSession(aUri, targetWindow.getSession().isPrivateMode()); session.setParentSession(targetWindow.getSession()); diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 947a9bab9f2..2f5c89bf349 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -43,7 +43,7 @@ diff --git a/app/src/main/assets/extensions/wolvic_autowebxr/content.js b/app/src/main/assets/extensions/wolvic_autowebxr/content.js new file mode 100644 index 00000000000..a29a97d3d0f --- /dev/null +++ b/app/src/main/assets/extensions/wolvic_autowebxr/content.js @@ -0,0 +1,152 @@ +const LOGTAG = '[wolvic:autowebxr]'; +const ENABLE_LOGS = true; +const logDebug = (...args) => ENABLE_LOGS && console.log(LOGTAG, ...args); + +const PARENT_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-parentElementXPath'; +const TARGET_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-targetElementXPath'; + +const IFRAME_READY_MSG = 'wolvic-autowebxr-iframeReady'; +const TARGET_ELEMENT_MSG = 'wolvic-autowebxr-targetElement'; + +var parentElementXPath; +var targetElementXPath; + +function getElementByXPath(document, xpath) { + let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null); + return result.singleNodeValue; +} + +// Limit the number of times that we can try to launch the experience, to avoid an infinite loop. +var retryCounter = 0; +const RETRY_LIMIT = 20; +function retryAfterTimeout(code, delay) { + if (retryCounter < RETRY_LIMIT) { + retryCounter++; + setTimeout(code, delay); + } else { + logDebug('Retry limit reached, will not try again'); + } +} + +function clickImmersiveElement() { + // Check if the current URL has extra query parameters + parentElementXPath = undefined; + targetElementXPath = undefined; + + let url = document.URL; + let params = new URLSearchParams(new URL(url).search); + for (let [key, value] of params) { + if (key === PARENT_ELEMENT_XPATH_PARAMETER) + parentElementXPath = value; + else if (key === TARGET_ELEMENT_XPATH_PARAMETER) + targetElementXPath = value; + } + + // We need at least the target element to click + if (!targetElementXPath) + return; + + logDebug('Preparing to open immersive WebXR; parentElementXPath: ' + parentElementXPath + ' ; targetElementXPath: ' + targetElementXPath); + + // The parent element is typically an iframe and, if it comes from a different origin, + // we might not be able to access its contents directly. + // If parentElementXPath is null, we will use the root to find the target element. + + var parent, parentDocument; + if (parentElementXPath) { + parent = getElementByXPath(document, parentElementXPath); + if (!parent) { + logDebug('Parent element not found, retrying'); + retryAfterTimeout(clickImmersiveElement, 1000); + return; + } + + try { + parentDocument = parent.contentDocument || parent.contentWindow.document; + } catch (e) { + logDebug('Parent iframe is from a different origin'); + const iframeWindow = parent.contentWindow; + + const targetElementMsg = { + action: TARGET_ELEMENT_MSG, + targetElementXPath: targetElementXPath + }; + + iframeWindow.postMessage(targetElementMsg, '*'); + + // The iframe might not be ready yet, so we set up a listener for the "iframe ready" message. + const handleIframeReady = (event) => { + if (event.source === iframeWindow && event.data === IFRAME_READY_MSG) { + window.removeEventListener('message', handleIframeReady); + iframeWindow.postMessage(targetElementMsg, '*'); + } + }; + window.addEventListener('message', handleIframeReady); + + return; + } + } else { + parent = window; + parentDocument = document; + } + + if (parentDocument.readyState !== 'complete') { + logDebug('Parent is still loading'); + parent.addEventListener('load', function() { + logDebug('Parent has finished loading'); + clickImmersiveElement(); + }); + return; + } else { + logDebug('Parent is loaded'); + } + + let targetElement = getElementByXPath(parentDocument, targetElementXPath); + + if (targetElement) { + logDebug('Target element found, calling click()'); + targetElement.click(); + } else { + logDebug('Target element not found, we will try again'); + retryAfterTimeout(clickImmersiveElement, 1000); + } +} + +function launchImmersiveFromIframe() { + window.addEventListener('message', function(event) { + if (event.data.action === TARGET_ELEMENT_MSG) { + let targetElement = getElementByXPath(document, event.data.targetElementXPath); + + if (targetElement) { + logDebug('Target element found in iframe, calling click()'); + targetElement.click(); + } else { + logDebug('Target element not found in iframe, retrying'); + retryAfterTimeout(function() { + window.postMessage(event.data, '*'); + }, 1000); + } + } + }); + + window.parent.postMessage(IFRAME_READY_MSG, '*'); +} + +// Main script execution +if (window.top === window.self) { + if (document.readyState === 'complete') { + logDebug('Root document is completely ready'); + clickImmersiveElement(); + } else { + logDebug('Root document is not ready yet'); + window.addEventListener('load', clickImmersiveElement); + } +} else { + if (document.readyState === 'complete') { + logDebug('Iframe is completely ready'); + launchImmersiveFromIframe(); + } else { + logDebug('Iframe is not ready yet'); + window.addEventListener('load', launchImmersiveFromIframe); + } +} diff --git a/app/src/main/assets/extensions/wolvic_autowebxr/manifest.json b/app/src/main/assets/extensions/wolvic_autowebxr/manifest.json new file mode 100644 index 00000000000..fe63297937b --- /dev/null +++ b/app/src/main/assets/extensions/wolvic_autowebxr/manifest.json @@ -0,0 +1,23 @@ +{ + "manifest_version": 2, + "name": "Wolvic WebXR Automator", + "version": "1.0", + "description": "Enter WebXR Experiences Automatically", + "browser_specific_settings": { + "gecko": { + "id": "wolvic-autowebxr@igalia.com" + } + }, + "content_scripts": [ + { + "matches": [ + "*://*/*" + ], + "js": [ + "content.js" + ], + "run_at": "document_idle", + "all_frames": true + } + ] +} diff --git a/app/src/main/res/raw/fxr_config.yaml b/app/src/main/res/raw/fxr_config.yaml index a70956746eb..381e87e72f4 100644 --- a/app/src/main/res/raw/fxr_config.yaml +++ b/app/src/main/res/raw/fxr_config.yaml @@ -26,4 +26,6 @@ prefs: browser.gesture.pinch.in: '' browser.gesture.pinch.out.shift: '' browser.gesture.pinch.in.shift: '' - apz.one_touch_pinch.enabled: false + apz.one_touch_pinch.enabled: false, + # allows scripts to open WebXR immersive sessions + dom.vr.require-gesture: false