From 4426c87d729e99a54dc09db08181af23fb0bcbbb Mon Sep 17 00:00:00 2001 From: Carlos Alberto Lopez Perez Date: Fri, 18 Oct 2024 15:43:17 +0100 Subject: [PATCH] Switch webkitgtk_minibrowser nightly runs to use a new universal bundle. The runs on the CI (TaskCluster) of WebKitGTK have been not working since a while [1] because the nightly bundles that we were shipping for download [2] for testing on the WPT CI stopped working due to binary compatibility issues. So we decided [3] that instead of shipping a bundle for a specific distro we would ship an universal bundle that can work on any distro. This allows us to ship a bundle with all the preview features enabled and the last versions of all the libraries, matching the same configurations that are tested on the release WebKitGTK bots at build.webkit.org This commit changes the taskcluster init tasks, as also the installation of the webkitgtk_minibrowser bundles to use this new universal bundle. This also allows now developers to test with webkitgtk_minibrowser nightlies independent of the distro they have. They simply need to pass the flags "--install-browser --channel=nightly" to "wpt run" [1] Fixes: https://github.com/web-platform-tests/wpt/issues/47823 [2] https://webkitgtk.org/built-products/x86_64/release/nightly [3] https://bugs.webkit.org/show_bug.cgi?id=280523 --- tools/ci/run_tc.py | 15 ++++++++---- tools/wpt/browser.py | 55 +++++++++++++++++--------------------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py index 446afadac88862..f3be8f85049f56 100755 --- a/tools/ci/run_tc.py +++ b/tools/ci/run_tc.py @@ -141,10 +141,15 @@ def install_certificates(): def start_dbus(): + # Start system bus run(["sudo", "service", "dbus", "start"]) - # Enable dbus autolaunch for Chrome - # https://source.chromium.org/chromium/chromium/src/+/main:content/app/content_main.cc;l=220;drc=0bcc023b8cdbc073aa5c48db373810db3f765c87. - os.environ["DBUS_SESSION_BUS_ADDRESS"] = "autolaunch:" + # Start user bus and set env + dbus_env = run(["dbus-launch"], return_stdout=True) + for dbus_env_line in dbus_env.splitlines(): + dbus_env_name, dbus_env_value = dbus_env_line.split('=', 1) + assert(dbus_env_name.startswith('DBUS_SESSION')) + os.environ[dbus_env_name] = dbus_env_value + assert ('DBUS_SESSION_BUS_ADDRESS' in os.environ) def install_chrome(channel): @@ -267,7 +272,9 @@ def setup_environment(args): if "chrome" in args.browser: assert args.channel is not None install_chrome(args.channel) - # Chrome is using dbus for various features. + + # This browsers use dbus for various features. + if any(b in args.browser for b in ["chrome", "webkitgtk_minibrowser"]): start_dbus() if args.xvfb: diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py index 932604397013ba..6e9338cc4807a9 100644 --- a/tools/wpt/browser.py +++ b/tools/wpt/browser.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta, timezone from shutil import which from typing import Optional -from urllib.parse import urlsplit +from urllib.parse import urlsplit, quote import html5lib import requests @@ -2375,34 +2375,20 @@ def version(self, binary=None, webdriver_binary=None): class WebKitGTKMiniBrowser(WebKit): - def _get_osidversion(self): - with open('/etc/os-release') as osrelease_handle: - for line in osrelease_handle.readlines(): - if line.startswith('ID='): - os_id = line.split('=')[1].strip().strip('"') - if line.startswith('VERSION_ID='): - version_id = line.split('=')[1].strip().strip('"') - assert os_id - assert version_id - osidversion = os_id + '-' + version_id - assert ' ' not in osidversion - assert len(osidversion) > 3 - return osidversion.capitalize() - def download(self, dest=None, channel=None, rename=None): base_dowload_uri = "https://webkitgtk.org/built-products/" - base_download_dir = base_dowload_uri + "x86_64/release/" + channel + "/" + self._get_osidversion() + "/MiniBrowser/" + base_download_dir = base_dowload_uri + platform.machine() + "/release/" + channel + "/MiniBrowser/" try: response = get(base_download_dir + "LAST-IS") except requests.exceptions.HTTPError as e: if e.response.status_code == 404: raise RuntimeError("Can't find a WebKitGTK MiniBrowser %s bundle for %s at %s" - % (channel, self._get_osidversion(), base_dowload_uri)) + % (channel, platform.machine(), base_dowload_uri)) raise bundle_filename = response.text.strip() - bundle_url = base_download_dir + bundle_filename + bundle_url = base_download_dir + quote(bundle_filename) dest = self._get_browser_download_dir(dest, channel) bundle_file_path = os.path.join(dest, bundle_filename) @@ -2411,13 +2397,14 @@ def download(self, dest=None, channel=None, rename=None): with open(bundle_file_path, "w+b") as f: get_download_to_descriptor(f, bundle_url) - bundle_filename_no_ext, _ = os.path.splitext(bundle_filename) + ext_ndots = 2 if '.tar.' in bundle_filename else 1 + bundle_filename_no_ext = '.'.join(bundle_filename.split('.')[:-ext_ndots]) bundle_hash_url = base_download_dir + bundle_filename_no_ext + ".sha256sum" bundle_expected_hash = get(bundle_hash_url).text.strip().split(" ")[0] bundle_computed_hash = sha256sum(bundle_file_path) if bundle_expected_hash != bundle_computed_hash: - self.logger.error("Calculated SHA256 hash is %s but was expecting %s" % (bundle_computed_hash,bundle_expected_hash)) + self.logger.error("Calculated SHA256 hash is %s but was expecting %s" % (bundle_computed_hash, bundle_expected_hash)) raise RuntimeError("The WebKitGTK MiniBrowser bundle at %s has incorrect SHA256 hash." % bundle_file_path) return bundle_file_path @@ -2431,26 +2418,26 @@ def install(self, dest=None, channel=None, prompt=True): rmtree(bundle_uncompress_directory) os.mkdir(bundle_uncompress_directory) + bundle_file_name = os.path.basename(bundle_path) with open(bundle_path, "rb") as f: - unzip(f, bundle_uncompress_directory) + if bundle_file_name.endswith(".zip"): + unzip(f, bundle_uncompress_directory) + elif ".tar." in bundle_file_name: + untar(f, bundle_uncompress_directory) + else: + raise NotImplementedError("Unable to install WebKitGTK MiniBrowser bundle from file:" % bundle_file_name) + os.remove(bundle_path) - install_dep_script = os.path.join(bundle_uncompress_directory, "install-dependencies.sh") - if os.path.isfile(install_dep_script): - self.logger.info("Executing install-dependencies.sh script from bundle.") - install_dep_cmd = [install_dep_script] - if not prompt: - install_dep_cmd.append("--autoinstall") - # use subprocess.check_call() directly to display unbuffered stdout/stderr in real-time. - subprocess.check_call(install_dep_cmd) + for expected_binary in ["MiniBrowser", "WebKitWebDriver"]: + binary_path = os.path.join(bundle_uncompress_directory, expected_binary) + if not (os.path.isfile(binary_path) and os.access(binary_path, os.X_OK)): + raise RuntimeError("Can't find a %s binary at %s" % (expected_binary, binary_path)) minibrowser_path = os.path.join(bundle_uncompress_directory, "MiniBrowser") - if not os.path.isfile(minibrowser_path): - raise RuntimeError("Can't find a MiniBrowser binary at %s" % minibrowser_path) - - os.remove(bundle_path) + version_str = subprocess.check_output([minibrowser_path, "--version"]).decode("utf-8").strip() + self.logger.info("WebKitGTK MiniBrowser bundle for channel %s installed: %s" % (channel, version_str)) install_ok_file = os.path.join(bundle_uncompress_directory, ".installation-ok") open(install_ok_file, "w").close() # touch - self.logger.info("WebKitGTK MiniBrowser bundle for channel %s installed." % channel) return minibrowser_path def _find_executable_in_channel_bundle(self, binary, venv_path=None, channel=None):