From cfe07aab6ca919a2d7f1f4e6c66a0ef940aef20c Mon Sep 17 00:00:00 2001 From: Leonardo Bartoli Date: Tue, 6 Aug 2024 15:45:20 +0200 Subject: [PATCH 1/5] Update selenium version used and firefox log parser --- internal/support/firefox_log_parser.py | 22 +++++++++++++++++++++- wptagent.py | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/support/firefox_log_parser.py b/internal/support/firefox_log_parser.py index 31787c6d7..079f0fa0f 100644 --- a/internal/support/firefox_log_parser.py +++ b/internal/support/firefox_log_parser.py @@ -296,6 +296,14 @@ def socket_thread_http_entry(self, msg): socket = self.http['current_socket'] self.http['connections'][connection] = {'socket': socket} del self.http['current_socket'] + elif msg['message'].startswith('TlsHandshaker::SetupSSL '): + match = re.search(r'^TlsHandshaker::SetupSSL (?P[\w\d]+)', + msg['message']) + if match: + connection = match.groupdict().get('connection') + if connection in self.http['connections']: + if 'ssl_start' not in self.http['connections'][connection]: + self.http['connections'][connection]['ssl_start'] = msg['timestamp'] elif msg['message'].startswith('nsHttpConnection::SetupSSL '): match = re.search(r'^nsHttpConnection::SetupSSL (?P[\w\d]+)', msg['message']) @@ -332,6 +340,17 @@ def socket_thread_http_entry(self, msg): if byte_count > 0 and trans_id in self.http['requests'] and \ 'start' not in self.http['requests'][trans_id]: self.http['requests'][trans_id]['start'] = msg['timestamp'] + elif msg['message'].startswith('nsHttpTransaction::OnSocketStatus ') and \ + msg['message'].find(' status=4b0005 progress=') > -1: + match = re.search(r'^nsHttpTransaction::OnSocketStatus ' + r'\[this=(?P[\w\d]+) status=4b0005 progress=(?P[\d+]+)', + msg['message']) + if match: + trans_id = match.groupdict().get('id') + byte_count = int(match.groupdict().get('bytes')) + if byte_count > 0 and trans_id in self.http['requests'] and \ + 'start' not in self.http['requests'][trans_id]: + self.http['requests'][trans_id]['start'] = msg['timestamp'] elif msg['message'].startswith('nsHttpTransaction::ProcessData '): match = re.search(r'^nsHttpTransaction::ProcessData \[this=(?P[\w\d]+)', msg['message']) @@ -446,6 +465,7 @@ def socket_transport_entry(self, msg): port = match.groupdict().get('port') self.http['sockets'][socket] = {'host': host, 'port': port} # nsSocketTransport::SendStatus [this=143f4000 status=804b0007] + # nsSocketTransport::SendStatus [this=7fe074bd2a00 status=4B0007] elif msg['message'].startswith('nsSocketTransport::SendStatus '): match = re.search(r'^nsSocketTransport::SendStatus \[' r'this=(?P[\w\d]+) ' @@ -453,7 +473,7 @@ def socket_transport_entry(self, msg): if match: socket = match.groupdict().get('socket') status = match.groupdict().get('status') - if status == '804b0007': + if status in ['804b0007', '4b0007']: if socket not in self.http['sockets']: self.http['sockets'][socket] = {} if 'start' not in self.http['sockets'][socket]: diff --git a/wptagent.py b/wptagent.py index 4ab82ba78..d28bec0b5 100644 --- a/wptagent.py +++ b/wptagent.py @@ -1084,7 +1084,7 @@ def fix_selenium_version(): newer versions are going to use 4.8.3 """ from internal.os_util import run_elevated - version = '4.8.3' + version = '4.18.1' if sys.version_info[1] == 6: version = '3.141.0' From 2035f8a1a00d674c2a465b793db01537380fafaa Mon Sep 17 00:00:00 2001 From: Leonardo Bartoli Date: Tue, 6 Aug 2024 16:12:14 +0200 Subject: [PATCH 2/5] Disable firefox OCSP --- internal/support/Firefox/profile/prefs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/support/Firefox/profile/prefs.js b/internal/support/Firefox/profile/prefs.js index 35c6eb9b5..b5ab7ed19 100644 --- a/internal/support/Firefox/profile/prefs.js +++ b/internal/support/Firefox/profile/prefs.js @@ -111,6 +111,7 @@ user_pref("security.warn_viewing_mixed", false); user_pref("security.warn_entering_secure", false); user_pref("security.warn_leaving_secure", false); user_pref("security.warn_submit_insecure", false); +user_pref("security.OCSP.enabled", 0); user_pref("services.settings.server", ""); user_pref("services.sync.migrated", true); user_pref("services.sync.engine.bookmarks", false); From 53e531a4117956c7dfdc644a5555ffb80ab7eb65 Mon Sep 17 00:00:00 2001 From: Leonardo Bartoli Date: Tue, 6 Aug 2024 16:28:18 +0200 Subject: [PATCH 3/5] Update Firefox browser class to support latest selenium version --- internal/firefox.py | 80 ++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/internal/firefox.py b/internal/firefox.py index 1daf2d53b..e3c1c1f05 100644 --- a/internal/firefox.py +++ b/internal/firefox.py @@ -31,6 +31,8 @@ import json from .desktop_browser import DesktopBrowser +def _get_location_uri(accuracy, lat, lng) -> str: + return f'data:application/json, {{ "status":"OK", "accuracy":{accuracy}, "location":{{ "lat":{lat}, "lng":{lng} }} }}' class Firefox(DesktopBrowser): """Firefox""" @@ -140,26 +142,47 @@ def start_firefox(self, job, task): return from selenium import webdriver # pylint: disable=import-error - capabilities = webdriver.DesiredCapabilities.FIREFOX.copy() - if 'ignoreSSL' in job and job['ignoreSSL']: - capabilities['acceptInsecureCerts'] = True - else: - capabilities['acceptInsecureCerts'] = False + if webdriver.__version__ >= "4.12": + service_args = ["--marionette-port", "2828"] + service = webdriver.FirefoxService(service_args=service_args, log_output=os.environ["MOZ_LOG_FILE"]) - capabilities['moz:firefoxOptions'] = { - 'binary': self.path, - 'args': ['-profile', task['profile']], - 'prefs': self.prepare_prefs(), - "log": {"level": "error"}, - 'env': { - "MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"], - "MOZ_LOG": os.environ["MOZ_LOG"] - } - } - service_args = ["--marionette-port", "2828"] + options = webdriver.FirefoxOptions() + options.binary_location = self.path + options.add_argument('--profile') + options.add_argument(f'{task["profile"]}') + options.log.level = 'error' + options.prefs = self.prepare_prefs() + + capabilities = webdriver.DesiredCapabilities.FIREFOX.copy() + if 'ignoreSSL' in job and job['ignoreSSL']: + capabilities['acceptInsecureCerts'] = True + else: + capabilities['acceptInsecureCerts'] = False + + for key, value in capabilities.items(): + options.set_capability(key, value) + self.driver = webdriver.Firefox(options=options, service=service) + elif webdriver.__version__ <= "4.9": + capabilities = webdriver.DesiredCapabilities.FIREFOX.copy() + if 'ignoreSSL' in job and job['ignoreSSL']: + capabilities['acceptInsecureCerts'] = True + else: + capabilities['acceptInsecureCerts'] = False - self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args) - logging.debug(self.driver.capabilities) + capabilities['moz:firefoxOptions'] = { + 'binary': self.path, + 'args': ['-profile', task['profile']], + 'prefs': self.prepare_prefs(), + "log": {"level": "error"}, + 'env': { + "MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"], + "MOZ_LOG": os.environ["MOZ_LOG"] + } + } + service_args = ["--marionette-port", "2828"] + self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args) + else: + raise Exception("Unsupported selenium version %s", webdriver.__version__) self.driver.set_page_load_timeout(task['time_limit']) if 'browserVersion' in self.driver.capabilities: @@ -208,17 +231,13 @@ def launch(self, job, task): ua_string += ' ' + task['AppendUA'] modified = True if modified: - logging.debug(ua_string) self.driver_set_pref('general.useragent.override', ua_string) # Location if 'lat' in self.job and 'lng' in self.job: try: lat = float(str(self.job['lat'])) lng = float(str(self.job['lng'])) - location_uri = 'data:application/json,{{'\ - '"status":"OK","accuracy":10.0,'\ - '"location":{{"lat":{0:f},"lng":{1:f}}}'\ - '}}'.format(lat, lng) + location_uri = _get_location_uri(10, lat, lng) logging.debug('Setting location: %s', location_uri) self.driver_set_pref('geo.wifi.uri', location_uri) except Exception: @@ -292,6 +311,8 @@ def close_browser(self, job, task): if platform.system() == "Linux": subprocess.call(['killall', '-9', 'firefox']) subprocess.call(['killall', '-9', 'firefox-trunk']) + subprocess.call(['killall', '-9', 'firefox-nightly']) + subprocess.call(['killall', '-9', 'firefox-esr']) os.environ["MOZ_LOG_FILE"] = '' os.environ["MOZ_LOG"] = '' @@ -326,7 +347,7 @@ def run_axe(self, task): script += "'" + "', '".join(axe_cats) + "'" script += ']}).then(results=>{return results;});' except Exception as err: - logging.exception("Exception running Axe: %s", err.__str__()) + logging.exception("Exception running Axe: %s", err) if self.must_exit_now: return completed = False @@ -349,7 +370,7 @@ def run_axe(self, task): axe_info['incomplete'] = axe_results['incomplete'] task['page_data']['axe'] = axe_info except Exception as err: - logging.exception("Exception running Axe: %s", err.__str__()) + logging.exception("Exception running Axe: %s", err) if not completed: task['page_data']['axe_failed'] = 1 self.axe_time = monotonic() - start @@ -376,7 +397,7 @@ def run_task(self, task): logging.exception("Exception running task") if command['record']: self.wait_for_page_load() - if not task['combine_steps'] or not len(task['script']): + if not task['combine_steps'] or not task['script']: self.on_stop_capture(task) self.on_stop_recording(task) recording = False @@ -397,10 +418,9 @@ def run_task(self, task): self.task = None def alert_size(self, _alert_config, _task_dir, _prefix): - '''Checks the agents file size and alert on certain percentage over avg byte size''' + '''Checks the agents file size and alert on certain percentage over avg byte size''' self.alert_desktop_results(_alert_config, 'Firefox', _task_dir, _prefix) - def wait_for_extension(self): """Wait for the extension to send the started message""" if self.job['message_server'] is not None: @@ -506,7 +526,7 @@ def run_js_file(self, file_name): script = None script_file_path = os.path.join(self.script_dir, file_name) if os.path.isfile(script_file_path): - with open(script_file_path, 'r') as script_file: + with open(script_file_path, 'r', encoding='utf-8') as script_file: script = script_file.read() if self.driver is not None and script is not None: try: @@ -518,7 +538,7 @@ def run_js_file(self, file_name): logging.debug(ret) return ret - def get_sorted_requests_json(self, include_bodies): + def get_sorted_requests_json(self, _include_bodies): return 'null' def collect_browser_metrics(self, task): From 80e0cd3b7ff7339e61ce17205e6791832b5d6a87 Mon Sep 17 00:00:00 2001 From: Leonardo Bartoli Date: Tue, 6 Aug 2024 16:40:58 +0200 Subject: [PATCH 4/5] Small refactoring --- internal/firefox.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/firefox.py b/internal/firefox.py index e3c1c1f05..3d183e46f 100644 --- a/internal/firefox.py +++ b/internal/firefox.py @@ -982,10 +982,7 @@ def process_command(self, command): parts = command['target'].split(',') lat = float(parts[0]) lng = float(parts[1]) - location_uri = 'data:application/json,{{'\ - '"status":"OK","accuracy":{2:d},'\ - '"location":{{"lat":{0:f},"lng":{1:f}}}'\ - '}}'.format(lat, lng, accuracy) + location_uri = _get_location_uri(accuracy, lat, lng) logging.debug('Setting location: %s', location_uri) self.set_pref('geo.wifi.uri', location_uri) except Exception: From 9172c6bef7ac0cfbdebc7dfec7c0b41b206c038b Mon Sep 17 00:00:00 2001 From: Leonardo Bartoli <142893024+lbartoli79@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:53:01 +0200 Subject: [PATCH 5/5] Update prefs.js --- internal/support/Firefox/profile/prefs.js | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/support/Firefox/profile/prefs.js b/internal/support/Firefox/profile/prefs.js index b5ab7ed19..35c6eb9b5 100644 --- a/internal/support/Firefox/profile/prefs.js +++ b/internal/support/Firefox/profile/prefs.js @@ -111,7 +111,6 @@ user_pref("security.warn_viewing_mixed", false); user_pref("security.warn_entering_secure", false); user_pref("security.warn_leaving_secure", false); user_pref("security.warn_submit_insecure", false); -user_pref("security.OCSP.enabled", 0); user_pref("services.settings.server", ""); user_pref("services.sync.migrated", true); user_pref("services.sync.engine.bookmarks", false);