Merge pull request #2797 from ClearlyClaire/glitch-soc/merge-upstream

Merge upstream changes up to 887e64e

app/controllers/application_controller.rb

@@ -23,7 +23,6 @@ class ApplicationController < ActionController::Base
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
-  helper_method :omniauth_only?
   helper_method :sso_account_settings
   helper_method :limited_federation_mode?
   helper_method :body_class_string
@@ -140,10 +139,6 @@ def use_seamless_external_login?
     Devise.pam_authentication || Devise.ldap_authentication
   end
 
-  def omniauth_only?
-    ENV['OMNIAUTH_ONLY'] == 'true'
-  end
-
   def sso_account_settings
     ENV.fetch('SSO_ACCOUNT_SETTINGS', nil)
   end

app/lib/link_details_extractor.rb

@@ -101,7 +101,7 @@ def root_array(root)
     end
 
     def json
-      @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
+      @json ||= root_array(Oj.load(@data)).compact.find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
     end
   end
 

app/models/account.rb

@@ -68,7 +68,7 @@ class Account < ApplicationRecord
   INSTANCE_ACTOR_ID = -99
 
   USERNAME_RE   = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i
-  MENTION_RE    = %r{(?<![=/[:word:]])@((#{USERNAME_RE})(?:@[[:word:].-]+[[:word:]]+)?)}i
+  MENTION_RE    = %r{(?<![=/[:word:]])@((#{USERNAME_RE})(?:@[[:word:].-]+[[:word:]]+)?)}
   URL_PREFIX_RE = %r{\Ahttp(s?)://[^/]+}
   USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i
   USERNAME_LENGTH_LIMIT = 30

app/models/tag.rb

@@ -37,7 +37,7 @@ class Tag < ApplicationRecord
   HASHTAG_LAST_SEQUENCE = '([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_NAME_PAT = "#{HASHTAG_FIRST_SEQUENCE}|#{HASHTAG_LAST_SEQUENCE}"
 
-  HASHTAG_RE = %r{(?<![=/)\p{Alnum}])#(#{HASHTAG_NAME_PAT})}i
+  HASHTAG_RE = %r{(?<![=/)\p{Alnum}])#(#{HASHTAG_NAME_PAT})}
   HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
   HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]\u0E47-\u0E4E#{HASHTAG_SEPARATORS}]/
 

app/models/user.rb

@@ -117,6 +117,7 @@ class User < ApplicationRecord
   scope :pending, -> { where(approved: false) }
   scope :approved, -> { where(approved: true) }
   scope :confirmed, -> { where.not(confirmed_at: nil) }
+  scope :unconfirmed, -> { where(confirmed_at: nil) }
   scope :enabled, -> { where(disabled: false) }
   scope :disabled, -> { where(disabled: true) }
   scope :active, -> { confirmed.signed_in_recently.account_not_suspended }

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -16,7 +16,7 @@ def perform
   private
 
   def clean_unconfirmed_accounts!
-    User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', UNCONFIRMED_ACCOUNTS_MAX_AGE_DAYS.days.ago).reorder(nil).find_in_batches do |batch|
+    User.unconfirmed.where(confirmation_sent_at: ..UNCONFIRMED_ACCOUNTS_MAX_AGE_DAYS.days.ago).reorder(nil).find_in_batches do |batch|
       # We have to do it separately because of missing database constraints
       AccountModerationNote.where(target_account_id: batch.map(&:account_id)).delete_all
       Account.where(id: batch.map(&:account_id)).delete_all

config/initializers/twitter_regex.rb

@@ -9,7 +9,7 @@ def emoji_parsing_enabled
 
   class Regex
     REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}<>()?]/iou
-    REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}()?!*"'「」<>;:=,.$%\[\]~&|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou
+    REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}()?!*"'「」<>;:=,.$%\[\]~&|]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou
     REGEXEN[:valid_url_balanced_parens] = /
       \(
         (?:

db/schema.rb

@@ -194,8 +194,8 @@
     t.integer "avatar_storage_schema_version"
     t.integer "header_storage_schema_version"
     t.string "devices_url"
-    t.integer "suspension_origin"
     t.datetime "sensitized_at", precision: nil
+    t.integer "suspension_origin"
     t.boolean "trendable"
     t.datetime "reviewed_at", precision: nil
     t.datetime "requested_review_at", precision: nil
@@ -579,12 +579,12 @@
   end
 
   create_table "ip_blocks", force: :cascade do |t|
-    t.datetime "created_at", precision: nil, null: false
-    t.datetime "updated_at", precision: nil, null: false
-    t.datetime "expires_at", precision: nil
     t.inet "ip", default: "0.0.0.0", null: false
     t.integer "severity", default: 0, null: false
+    t.datetime "expires_at", precision: nil
     t.text "comment", default: "", null: false
+    t.datetime "created_at", precision: nil, null: false
+    t.datetime "updated_at", precision: nil, null: false
     t.index ["ip"], name: "index_ip_blocks_on_ip", unique: true
   end
 
@@ -1424,9 +1424,9 @@
   add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true
 
   create_view "user_ips", sql_definition: <<-SQL
-      SELECT t0.user_id,
-      t0.ip,
-      max(t0.used_at) AS used_at
+      SELECT user_id,
+      ip,
+      max(used_at) AS used_at
      FROM ( SELECT users.id AS user_id,
               users.sign_up_ip AS ip,
               users.created_at AS used_at
@@ -1443,7 +1443,7 @@
               login_activities.created_at
              FROM login_activities
             WHERE (login_activities.success = true)) t0
-    GROUP BY t0.user_id, t0.ip;
+    GROUP BY user_id, ip;
   SQL
   create_view "account_summaries", materialized: true, sql_definition: <<-SQL
       SELECT accounts.id AS account_id,
@@ -1464,9 +1464,9 @@
   add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true
 
   create_view "global_follow_recommendations", materialized: true, sql_definition: <<-SQL
-      SELECT t0.account_id,
-      sum(t0.rank) AS rank,
-      array_agg(t0.reason) AS reason
+      SELECT account_id,
+      sum(rank) AS rank,
+      array_agg(reason) AS reason
      FROM ( SELECT account_summaries.account_id,
               ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank,
               'most_followed'::text AS reason
@@ -1490,8 +1490,8 @@
                     WHERE (follow_recommendation_suppressions.account_id = statuses.account_id)))))
             GROUP BY account_summaries.account_id
            HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0
-    GROUP BY t0.account_id
-    ORDER BY (sum(t0.rank)) DESC;
+    GROUP BY account_id
+    ORDER BY (sum(rank)) DESC;
   SQL
   add_index "global_follow_recommendations", ["account_id"], name: "index_global_follow_recommendations_on_account_id", unique: true
 

spec/lib/link_details_extractor_spec.rb

@@ -129,6 +129,24 @@
       include_examples 'structured data'
     end
 
+    context 'with the first tag is null' do
+      let(:html) { <<~HTML }
+        <!doctype html>
+        <html>
+        <body>
+          <script type="application/ld+json">
+            null
+          </script>
+          <script type="application/ld+json">
+            #{ld_json}
+          </script>
+        </body>
+        </html>
+      HTML
+
+      include_examples 'structured data'
+    end
+
     context 'with preceding block of unsupported LD+JSON' do
       let(:html) { <<~HTML }
         <!doctype html>

spec/lib/text_formatter_spec.rb

@@ -224,6 +224,14 @@
       end
     end
 
+    context 'when given a URL with trailing @ symbol' do
+      let(:text) { 'https://gta.fandom.com/wiki/TW@ Content' }
+
+      it 'matches the full URL' do
+        expect(subject).to include 'href="https://gta.fandom.com/wiki/TW@"'
+      end
+    end
+
     context 'when given a URL containing unsafe code (XSS attack, visible part)' do
       let(:text) { 'http://example.com/b<del>b</del>' }
 

spec/models/account_spec.rb

@@ -711,6 +711,14 @@
     it 'does not match URL query string' do
       expect(subject.match('https://example.com/?x=@alice')).to be_nil
     end
+
+    it 'matches usernames immediately following the letter ß' do
+      expect(subject.match('Hello toß @alice from me')[1]).to eq 'alice'
+    end
+
+    it 'matches usernames containing uppercase characters' do
+      expect(subject.match('Hello to @[email protected] from me')[1]).to eq '[email protected]'
+    end
   end
 
   describe 'validations' do

spec/models/tag_spec.rb

@@ -95,6 +95,14 @@
     it 'does not match purely-numeric hashtags' do
       expect(subject.match('hello #0123456')).to be_nil
     end
+
+    it 'matches hashtags immediately following the letter ß' do
+      expect(subject.match('Hello toß #ruby').to_s).to eq '#ruby'
+    end
+
+    it 'matches hashtags containing uppercase characters' do
+      expect(subject.match('Hello #rubyOnRails').to_s).to eq '#rubyOnRails'
+    end
   end
 
   describe '#to_param' do

spec/support/matchers/api_pagination.rb

@@ -7,7 +7,7 @@
     end.all?
   end
 
-  failure_message do |header|
-    "expected that #{header} would have the same values as #{links}."
+  failure_message do |response|
+    "expected that #{response.headers['Link']} would have the same values as #{links}."
   end
 end

spec/workers/scheduler/user_cleanup_scheduler_spec.rb

@@ -12,29 +12,31 @@
 
   describe '#perform' do
     before do
-      # Need to update the already-existing users because their initialization overrides confirmation_sent_at
+      # Update already-existing users because initialization overrides `confirmation_sent_at`
       new_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: Time.now.utc)
       old_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: 10.days.ago)
       confirmed_user.update!(confirmed_at: 1.day.ago)
     end
 
-    it 'deletes the old unconfirmed user, their account, and the moderation note' do
+    it 'deletes the old unconfirmed user and metadata while preserving confirmed user and newer unconfirmed user' do
       expect { subject.perform }
-        .to change { User.exists?(old_unconfirmed_user.id) }.from(true).to(false)
-        .and change { Account.exists?(old_unconfirmed_user.account_id) }.from(true).to(false)
-      expect { moderation_note.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        .to change { User.exists?(old_unconfirmed_user.id) }
+        .from(true).to(false)
+        .and change { Account.exists?(old_unconfirmed_user.account_id) }
+        .from(true).to(false)
+      expect { moderation_note.reload }
+        .to raise_error(ActiveRecord::RecordNotFound)
+      expect_preservation_of(new_unconfirmed_user)
+      expect_preservation_of(confirmed_user)
     end
 
-    it 'does not delete the new unconfirmed user or their account' do
-      subject.perform
-      expect(User.exists?(new_unconfirmed_user.id)).to be true
-      expect(Account.exists?(new_unconfirmed_user.account_id)).to be true
-    end
+    private
 
-    it 'does not delete the confirmed user or their account' do
-      subject.perform
-      expect(User.exists?(confirmed_user.id)).to be true
-      expect(Account.exists?(confirmed_user.account_id)).to be true
+    def expect_preservation_of(user)
+      expect(User.exists?(user.id))
+        .to be true
+      expect(Account.exists?(user.account_id))
+        .to be true
     end
   end
 end