diff --git a/config/environments/test.rb b/config/environments/test.rb
index 58a84b28d1ee..1c1a2072f192 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -87,7 +87,7 @@
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
 
-  config.cache_store = :file_store, Rails.root.join("tmp", "cache", "paralleltests#{ENV['TEST_ENV_NUMBER']}")
+  config.cache_store = :file_store, Rails.root.join("tmp", "cache", "paralleltests#{ENV.fetch('TEST_ENV_NUMBER', nil)}")
 
   if ENV['TEST_ENV_NUMBER']
     assets_cache_path = Rails.root.join("tmp/cache/assets/paralleltests#{ENV['TEST_ENV_NUMBER']}")
@@ -96,4 +96,8 @@
 
   # Speed up tests by lowering BCrypt's cost function
   BCrypt::Engine.cost = BCrypt::Engine::MIN_COST
+
+  # Pin Chromedriver version to last version that was released the "old way"
+  # See https://github.com/titusfortner/webdrivers/issues/247
+  Webdrivers::Chromedriver.required_version = "114.0.5735.90"
 end
diff --git a/docker/ci/entrypoint.sh b/docker/ci/entrypoint.sh
index 28f312d38130..dd9c75d2d5cb 100755
--- a/docker/ci/entrypoint.sh
+++ b/docker/ci/entrypoint.sh
@@ -68,7 +68,7 @@ setup_tests() {
 	fi
 
 	# create test database "app" and dump schema because db/structure.sql is not checked in
-	execute_quiet "time bundle exec rails db:migrate db:schema:dump zeitwerk:check webdrivers:chromedriver:update webdrivers:geckodriver:update"
+	execute_quiet "time bundle exec rails db:migrate db:schema:dump zeitwerk:check webdrivers:chromedriver:update[114.0.5735.90] webdrivers:geckodriver:update"
 }
 
 run_units() {
diff --git a/script/ci/cache_prepare.sh b/script/ci/cache_prepare.sh
index 0f6a9ff5b3d7..a76ddc612b2a 100644
--- a/script/ci/cache_prepare.sh
+++ b/script/ci/cache_prepare.sh
@@ -37,7 +37,7 @@ run() {
   eval $2;
 }
 
-run "bundle exec rake db:create db:migrate webdrivers:chromedriver:update webdrivers:geckodriver:update"
+run "bundle exec rake db:create db:migrate webdrivers:chromedriver:update[114.0.5735.90] webdrivers:geckodriver:update"
 
 run "cd frontend; npm install ; cd -"