Merge upstream up to 425d77f8124a50fc033e8fb3bdf7b89a6a25f4fa

.github/workflows/test-ruby.yml

@@ -153,3 +153,100 @@ jobs:
         run: './bin/rails db:create db:schema:load db:seed'
 
       - run: bundle exec rake rspec_chunked
+
+  test-e2e:
+    name: End to End testing
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      BUNDLE_WITH: test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.0'
+          - '3.1'
+          - '.ruby-version'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Update package index
+        run: sudo apt-get update
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+          node-version-file: '.nvmrc'
+
+      - name: Install native Ruby dependencies
+        run: sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Install additional system dependencies
+        run: sudo apt-get install -y ffmpeg imagemagick
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          bundler-cache: true
+
+      - run: yarn --frozen-lockfile
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bundle exec rake spec:system
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: e2e-logs-${{ matrix.ruby-version }}
+          path: log/
+
+      - name: Archive test screenshots
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: e2e-screenshots
+          path: tmp/screenshots/

.rubocop.yml

@@ -38,14 +38,7 @@ Layout/FirstHashElementIndentation:
 # Reason: Currently disabled in .rubocop_todo.yml
 # https://docs.rubocop.org/rubocop/cops_layout.html#layoutlinelength
 Layout/LineLength:
-  AllowedPatterns:
-    # Allow comments to be long lines
-    - !ruby/regexp / \# .*$/
-    - !ruby/regexp /^\# .*$/
-  Exclude:
-    - 'lib/mastodon/cli/*.rb'
-    - db/*migrate/**/*
-    - db/seeds/**/*
+  Max: 320 # Default of 120 causes a duplicate entry in generated todo file
 
 # Reason:
 # https://docs.rubocop.org/rubocop/cops_lint.html#lintuselessaccessmodifier

.rubocop_todo.yml

@@ -39,6 +39,13 @@ Layout/LeadingCommentSpace:
     - 'config/application.rb'
     - 'config/initializers/omniauth.rb'
 
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Exclude:
+    - 'app/models/account.rb'
+
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: require_no_space, require_space
@@ -112,7 +119,6 @@ Lint/UselessAssignment:
     - 'config/initializers/omniauth.rb'
     - 'db/migrate/20190511134027_add_silenced_at_suspended_at_to_accounts.rb'
     - 'db/post_migrate/20190511152737_remove_suspended_silenced_account_fields.rb'
-    - 'spec/controllers/api/v1/bookmarks_controller_spec.rb'
     - 'spec/controllers/api/v1/favourites_controller_spec.rb'
     - 'spec/controllers/concerns/account_controller_concern_spec.rb'
     - 'spec/helpers/jsonld_helper_spec.rb'
@@ -129,7 +135,7 @@ Lint/UselessAssignment:
 
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
 Metrics/AbcSize:
-  Max: 150
+  Max: 143
   Exclude:
     - 'app/serializers/initial_state_serializer.rb'
 
@@ -160,14 +166,6 @@ Naming/VariableNumber:
     - 'spec/models/domain_block_spec.rb'
     - 'spec/models/user_spec.rb'
 
-# This cop supports unsafe autocorrection (--autocorrect-all).
-Performance/UnfreezeString:
-  Exclude:
-    - 'app/lib/rss/builder.rb'
-    - 'app/lib/text_formatter.rb'
-    - 'app/validators/status_length_validator.rb'
-    - 'lib/tasks/mastodon.rake'
-
 RSpec/AnyInstance:
   Exclude:
     - 'spec/controllers/activitypub/inboxes_controller_spec.rb'
@@ -187,41 +185,6 @@ RSpec/AnyInstance:
     - 'spec/workers/activitypub/delivery_worker_spec.rb'
     - 'spec/workers/web/push_notification_worker_spec.rb'
 
-# This cop supports unsafe autocorrection (--autocorrect-all).
-RSpec/EmptyExampleGroup:
-  Exclude:
-    - 'spec/helpers/admin/action_logs_helper_spec.rb'
-    - 'spec/models/account_alias_spec.rb'
-    - 'spec/models/account_deletion_request_spec.rb'
-    - 'spec/models/account_moderation_note_spec.rb'
-    - 'spec/models/announcement_mute_spec.rb'
-    - 'spec/models/announcement_reaction_spec.rb'
-    - 'spec/models/announcement_spec.rb'
-    - 'spec/models/backup_spec.rb'
-    - 'spec/models/conversation_mute_spec.rb'
-    - 'spec/models/custom_filter_keyword_spec.rb'
-    - 'spec/models/custom_filter_spec.rb'
-    - 'spec/models/device_spec.rb'
-    - 'spec/models/encrypted_message_spec.rb'
-    - 'spec/models/featured_tag_spec.rb'
-    - 'spec/models/follow_recommendation_suppression_spec.rb'
-    - 'spec/models/list_account_spec.rb'
-    - 'spec/models/list_spec.rb'
-    - 'spec/models/login_activity_spec.rb'
-    - 'spec/models/mute_spec.rb'
-    - 'spec/models/preview_card_spec.rb'
-    - 'spec/models/preview_card_trend_spec.rb'
-    - 'spec/models/relay_spec.rb'
-    - 'spec/models/scheduled_status_spec.rb'
-    - 'spec/models/status_stat_spec.rb'
-    - 'spec/models/status_trend_spec.rb'
-    - 'spec/models/system_key_spec.rb'
-    - 'spec/models/tag_follow_spec.rb'
-    - 'spec/models/unavailable_domain_spec.rb'
-    - 'spec/models/user_invite_request_spec.rb'
-    - 'spec/models/web/setting_spec.rb'
-    - 'spec/services/unmute_service_spec.rb'
-
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 22
@@ -354,43 +317,6 @@ Rails/ApplicationController:
   Exclude:
     - 'app/controllers/health_controller.rb'
 
-# Configuration parameters: Database, Include.
-# SupportedDatabases: mysql, postgresql
-# Include: db/**/*.rb
-Rails/BulkChangeTable:
-  Exclude:
-    - 'db/migrate/20160222143943_add_profile_fields_to_accounts.rb'
-    - 'db/migrate/20160223162837_add_metadata_to_statuses.rb'
-    - 'db/migrate/20160305115639_add_devise_to_users.rb'
-    - 'db/migrate/20160314164231_add_owner_to_application.rb'
-    - 'db/migrate/20160926213048_remove_owner_from_application.rb'
-    - 'db/migrate/20161003142332_add_confirmable_to_users.rb'
-    - 'db/migrate/20170112154826_migrate_settings.rb'
-    - 'db/migrate/20170127165745_add_devise_two_factor_to_users.rb'
-    - 'db/migrate/20170322143850_change_primary_key_to_bigint_on_statuses.rb'
-    - 'db/migrate/20170330021336_add_counter_caches.rb'
-    - 'db/migrate/20170425202925_add_oembed_to_preview_cards.rb'
-    - 'db/migrate/20170427011934_re_add_owner_to_application.rb'
-    - 'db/migrate/20170520145338_change_language_filter_to_opt_out.rb'
-    - 'db/migrate/20170624134742_add_description_to_session_activations.rb'
-    - 'db/migrate/20170718211102_add_activitypub_to_accounts.rb'
-    - 'db/migrate/20171006142024_add_uri_to_custom_emojis.rb'
-    - 'db/migrate/20180812123222_change_relays_enabled.rb'
-    - 'db/migrate/20190511134027_add_silenced_at_suspended_at_to_accounts.rb'
-    - 'db/migrate/20190805123746_add_capabilities_to_tags.rb'
-    - 'db/migrate/20190807135426_add_comments_to_domain_blocks.rb'
-    - 'db/migrate/20190815225426_add_last_status_at_to_tags.rb'
-    - 'db/migrate/20190901035623_add_max_score_to_tags.rb'
-    - 'db/migrate/20200417125749_add_storage_schema_version.rb'
-    - 'db/migrate/20200608113046_add_sign_in_token_to_users.rb'
-    - 'db/migrate/20211112011713_add_language_to_preview_cards.rb'
-    - 'db/migrate/20211231080958_add_category_to_reports.rb'
-    - 'db/migrate/20220202200743_add_trendable_to_accounts.rb'
-    - 'db/migrate/20220224010024_add_ips_to_email_domain_blocks.rb'
-    - 'db/migrate/20220227041951_add_last_used_at_to_oauth_access_tokens.rb'
-    - 'db/migrate/20220303000827_add_ordered_media_attachment_ids_to_status_edits.rb'
-    - 'db/migrate/20220824164433_add_human_identifier_to_admin_action_logs.rb'
-
 # Configuration parameters: Include.
 # Include: db/**/*.rb
 Rails/CreateTableWithTimestamps:
@@ -666,7 +592,7 @@ Style/FetchEnvVar:
     - 'app/lib/translation_service.rb'
     - 'config/environments/development.rb'
     - 'config/environments/production.rb'
-    - 'config/initializers/2_whitelist_mode.rb'
+    - 'config/initializers/2_limited_federation_mode.rb'
     - 'config/initializers/blacklists.rb'
     - 'config/initializers/cache_buster.rb'
     - 'config/initializers/content_security_policy.rb'
@@ -929,9 +855,3 @@ Style/WordArray:
     - 'config/initializers/cors.rb'
     - 'spec/controllers/settings/imports_controller_spec.rb'
     - 'spec/models/form/import_spec.rb'
-
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
-# URISchemes: http, https
-Layout/LineLength:
-  Max: 701

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.6] - 2023-07-31
+
+### Fixed
+
+- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228))
+- Fix wrong filters sometimes applying in streaming ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26159), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26213), [renchap](https://github.com/mastodon/mastodon/pull/26233))
+- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116))
+
 ## [4.1.5] - 2023-07-21
 
 ### Added

Gemfile

@@ -99,9 +99,6 @@ gem 'rdf-normalize', '~> 0.5'
 gem 'private_address_check', '~> 0.5'
 
 group :test do
-  # RSpec runner for rails
-  gem 'rspec-rails', '~> 6.0'
-
   # Used to split testing into chunks in CI
   gem 'rspec_chunked', '~> 0.6'
 
@@ -113,6 +110,10 @@ group :test do
 
   # Browser integration testing
   gem 'capybara', '~> 3.39'
+  gem 'selenium-webdriver'
+
+  # Used to reset the database between system tests
+  gem 'database_cleaner-active_record'
 
   # Used to mock environment variables
   gem 'climate_control', '~> 0.2'
@@ -173,10 +174,19 @@ group :development do
 
   # Validate missing i18n keys
   gem 'i18n-tasks', '~> 1.0', require: false
+end
 
+group :development, :test do
   # Profiling tools
   gem 'memory_profiler', require: false
+  gem 'ruby-prof', require: false
   gem 'stackprof', require: false
+  gem 'test-prof'
+end
+
+group :development, :test do
+  # RSpec runner for rails
+  gem 'rspec-rails', '~> 6.0'
 end
 
 group :production do

Gemfile.lock

@@ -199,6 +199,10 @@ GEM
     crass (1.0.6)
     css_parser (1.14.0)
       addressable
+    database_cleaner-active_record (2.1.0)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     date (3.3.3)
     debug_inspector (1.1.0)
     devise (4.9.2)
@@ -640,6 +644,7 @@ GEM
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
+    ruby-prof (1.6.3)
     ruby-progressbar (1.13.0)
     ruby-saml (1.15.0)
       nokogiri (>= 1.13.10)
@@ -656,6 +661,10 @@ GEM
     scenic (1.7.0)
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
+    selenium-webdriver (4.9.1)
+      rexml (~> 3.2, >= 3.2.5)
+      rubyzip (>= 1.2.2, < 3.0)
+      websocket (~> 1.0)
     semantic_range (3.0.0)
     sidekiq (6.5.9)
       connection_pool (>= 2.2.5, < 3)
@@ -710,6 +719,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
+    test-prof (1.2.1)
     thor (1.2.2)
     tilt (2.2.0)
     timeout (0.4.0)
@@ -768,6 +778,7 @@ GEM
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
+    websocket (1.2.9)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -804,6 +815,7 @@ DEPENDENCIES
   color_diff (~> 0.1)
   concurrent-ruby
   connection_pool
+  database_cleaner-active_record
   devise (~> 4.9)
   devise-two-factor (~> 4.1)
   devise_pam_authenticatable2 (~> 9.2)
@@ -881,10 +893,12 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rspec
+  ruby-prof
   ruby-progressbar (~> 1.13)
   rubyzip (~> 2.3)
   sanitize (~> 6.0)
   scenic (~> 1.7)
+  selenium-webdriver
   sidekiq (~> 6.5)
   sidekiq-bulk (~> 0.2.0)
   sidekiq-scheduler (~> 5.0)
@@ -897,6 +911,7 @@ DEPENDENCIES
   stackprof
   stoplight (~> 3.0.1)
   strong_migrations (~> 0.8)
+  test-prof
   thor (~> 1.2)
   tty-prompt (~> 0.23)
   twitter-text (~> 3.1.0)