Merge pull request #1513 from ClearlyClaire/glitch-soc/merge-upstream

Merge upstream changes

Gemfile

@@ -111,7 +111,7 @@ group :development, :test do
   gem 'i18n-tasks', '~> 0.9', require: false
   gem 'pry-byebug', '~> 3.9'
   gem 'pry-rails', '~> 0.3'
-  gem 'rspec-rails', '~> 4.1'
+  gem 'rspec-rails', '~> 5.0'
 end
 
 group :production, :test do
@@ -143,7 +143,7 @@ group :development do
   gem 'rubocop', '~> 1.11', require: false
   gem 'rubocop-rails', '~> 2.9', require: false
   gem 'brakeman', '~> 4.10', require: false
-  gem 'bundler-audit', '~> 0.7', require: false
+  gem 'bundler-audit', '~> 0.8', require: false
 
   gem 'capistrano', '~> 3.16'
   gem 'capistrano-rails', '~> 1.6'
@@ -155,7 +155,6 @@ end
 
 group :production do
   gem 'lograge', '~> 0.11'
-  gem 'redis-rails', '~> 5.0'
 end
 
 gem 'concurrent-ruby', require: false

Gemfile.lock

@@ -115,9 +115,9 @@ GEM
     bullet (6.1.4)
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.11)
-    bundler-audit (0.7.0.1)
+    bundler-audit (0.8.0)
       bundler (>= 1.2.0, < 3)
-      thor (>= 0.18, < 2)
+      thor (~> 1.0)
     byebug (11.1.3)
     capistrano (3.16.0)
       airbrussh (>= 1.0.0)
@@ -492,24 +492,8 @@ GEM
       rdf (~> 3.1)
     redcarpet (3.5.1)
     redis (4.2.5)
-    redis-actionpack (5.2.0)
-      actionpack (>= 5, < 7)
-      redis-rack (>= 2.1.0, < 3)
-      redis-store (>= 1.1.0, < 2)
-    redis-activesupport (5.2.0)
-      activesupport (>= 3, < 7)
-      redis-store (>= 1.3, < 2)
     redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    redis-rack (2.1.3)
-      rack (>= 2.0.8, < 3)
-      redis-store (>= 1.2, < 2)
-    redis-rails (5.0.2)
-      redis-actionpack (>= 5.0, < 6)
-      redis-activesupport (>= 5.0, < 6)
-      redis-store (>= 1.2, < 2)
-    redis-store (1.9.0)
-      redis (>= 4, < 5)
     regexp_parser (2.1.1)
     request_store (1.5.0)
       rack (>= 1.4)
@@ -531,10 +515,10 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-rails (4.1.0)
-      actionpack (>= 4.2)
-      activesupport (>= 4.2)
-      railties (>= 4.2)
+    rspec-rails (5.0.0)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
+      railties (>= 5.2)
       rspec-core (~> 3.10)
       rspec-expectations (~> 3.10)
       rspec-mocks (~> 3.10)
@@ -706,7 +690,7 @@ DEPENDENCIES
   brakeman (~> 4.10)
   browser
   bullet (~> 6.1)
-  bundler-audit (~> 0.7)
+  bundler-audit (~> 0.8)
   capistrano (~> 3.16)
   capistrano-rails (~> 1.6)
   capistrano-rbenv (~> 2.2)
@@ -792,9 +776,8 @@ DEPENDENCIES
   redcarpet (~> 3.5)
   redis (~> 4.2)
   redis-namespace (~> 1.8)
-  redis-rails (~> 5.0)
   rqrcode (~> 1.2)
-  rspec-rails (~> 4.1)
+  rspec-rails (~> 5.0)
   rspec-sidekiq (~> 3.1)
   rspec_junit_formatter (~> 0.4)
   rubocop (~> 1.11)

app/controllers/admin/domain_blocks_controller.rb

@@ -22,7 +22,7 @@ module Admin
       if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block)
         @domain_block.save
         flash.now[:alert] = I18n.t('admin.domain_blocks.existing_domain_block_html', name: existing_domain_block.domain, unblock_url: admin_domain_block_path(existing_domain_block)).html_safe # rubocop:disable Rails/OutputSafety
-        @domain_block.errors[:domain].clear
+        @domain_block.errors.delete(:domain)
         render :new
       else
         if existing_domain_block.present?

app/controllers/concerns/cache_concern.rb

@@ -31,7 +31,9 @@ module CacheConcern
   def cache_collection(raw, klass)
     return raw unless klass.respond_to?(:with_includes)
 
-    raw                    = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
+    raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
+    return [] if raw.empty?
+
     cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id)
     uncached_ids           = raw.map(&:id) - cached_keys_with_value.keys
 

app/javascript/flavours/glitch/actions/store.js

@@ -1,6 +1,7 @@
 import { Iterable, fromJS } from 'immutable';
 import { hydrateCompose } from './compose';
 import { importFetchedAccounts } from './importer';
+import { saveSettings } from './settings';
 
 export const STORE_HYDRATE = 'STORE_HYDRATE';
 export const STORE_HYDRATE_LAZY = 'STORE_HYDRATE_LAZY';
@@ -9,9 +10,22 @@ const convertState = rawState =>
   fromJS(rawState, (k, v) =>
     Iterable.isIndexed(v) ? v.toList() : v.toMap());
 
+const applyMigrations = (state) => {
+  return state.withMutations(state => {
+    // Migrate glitch-soc local-only “Show unread marker” setting to Mastodon's setting
+    if (state.getIn(['local_settings', 'notifications', 'show_unread']) !== undefined) {
+      // Only change if the Mastodon setting does not deviate from default
+      if (state.getIn(['settings', 'notifications', 'showUnread']) !== false) {
+        state.setIn(['settings', 'notifications', 'showUnread'], state.getIn(['local_settings', 'notifications', 'show_unread']));
+      }
+      state.removeIn(['local_settings', 'notifications', 'show_unread'])
+    }
+  });
+};
+
 export function hydrateStore(rawState) {
   return dispatch => {
-    const state = convertState(rawState);
+    const state = applyMigrations(convertState(rawState));
 
     dispatch({
       type: STORE_HYDRATE,
@@ -20,5 +34,6 @@ export function hydrateStore(rawState) {
 
     dispatch(hydrateCompose());
     dispatch(importFetchedAccounts(Object.values(rawState.accounts)));
+    dispatch(saveSettings());
   };
 };

app/javascript/flavours/glitch/features/local_settings/page/index.js

@@ -113,14 +113,6 @@ class LocalSettingsPage extends React.PureComponent {
             <FormattedMessage id='settings.notifications.favicon_badge' defaultMessage='Unread notifications favicon badge' />
             <span className='hint'><FormattedMessage id='settings.notifications.favicon_badge.hint' defaultMessage="Add a badge for unread notifications to the favicon" /></span>
           </LocalSettingsPageItem>
-          <LocalSettingsPageItem
-            settings={settings}
-            item={['notifications', 'show_unread']}
-            id='mastodon-settings--notifications-show_unread'
-            onChange={onChange}
-          >
-            <FormattedMessage id='settings.notifications.show_unread' defaultMessage='Show unread marker' />
-          </LocalSettingsPageItem>
         </section>
         <section>
           <h2><FormattedMessage id='settings.layout_opts' defaultMessage='Layout options' /></h2>

app/javascript/flavours/glitch/features/notifications/components/column_settings.js

@@ -56,6 +56,16 @@ export default class ColumnSettings extends React.PureComponent {
           <ClearColumnButton onClick={onClear} />
         </div>
 
+        <div role='group' aria-labelledby='notifications-unread-markers'>
+          <span id='notifications-unread-markers' className='column-settings__section'>
+            <FormattedMessage id='notifications.column_settings.unread_markers.category' defaultMessage='Unread notification markers' />
+          </span>
+
+          <div className='column-settings__row'>
+            <SettingToggle id='unread-notification-markers' prefix='notifications' settings={settings} settingPath={['showUnread']} onChange={onChange} label={filterShowStr} />
+          </div>
+        </div>
+
         <div role='group' aria-labelledby='notifications-filter-bar'>
           <span id='notifications-filter-bar' className='column-settings__section'>
             <FormattedMessage id='notifications.column_settings.filter_bar.category' defaultMessage='Quick filter bar' />

app/javascript/flavours/glitch/features/notifications/index.js

@@ -67,8 +67,8 @@ const mapStateToProps = state => ({
   hasMore: state.getIn(['notifications', 'hasMore']),
   numPending: state.getIn(['notifications', 'pendingItems'], ImmutableList()).size,
   notifCleaningActive: state.getIn(['notifications', 'cleaningMode']),
-  lastReadId: state.getIn(['local_settings', 'notifications', 'show_unread']) ? state.getIn(['notifications', 'readMarkerId']) : '0',
-  canMarkAsRead: state.getIn(['local_settings', 'notifications', 'show_unread']) && state.getIn(['notifications', 'readMarkerId']) !== '0' && getNotifications(state).some(item => item !== null && compareId(item.get('id'), state.getIn(['notifications', 'readMarkerId'])) > 0),
+  lastReadId: state.getIn(['settings', 'notifications', 'showUnread']) ? state.getIn(['notifications', 'readMarkerId']) : '0',
+  canMarkAsRead: state.getIn(['settings', 'notifications', 'showUnread']) && state.getIn(['notifications', 'readMarkerId']) !== '0' && getNotifications(state).some(item => item !== null && compareId(item.get('id'), state.getIn(['notifications', 'readMarkerId'])) > 0),
   needsNotificationPermission: state.getIn(['settings', 'notifications', 'alerts']).includes(true) && state.getIn(['notifications', 'browserSupport']) && state.getIn(['notifications', 'browserPermission']) === 'default' && !state.getIn(['settings', 'notifications', 'dismissPermissionBanner']),
 });
 

app/javascript/flavours/glitch/reducers/local_settings.js

@@ -55,7 +55,6 @@ const initialState = ImmutableMap({
   notifications : ImmutableMap({
     favicon_badge : false,
     tab_badge     : true,
-    show_unread   : true,
   }),
 });
 

app/javascript/flavours/glitch/reducers/settings.js

@@ -49,6 +49,7 @@ const initialState = ImmutableMap({
     }),
 
     dismissPermissionBanner: false,
+    showUnread: true,
 
     shows: ImmutableMap({
       follow: true,

app/javascript/mastodon/features/notifications/components/column_settings.js

@@ -55,6 +55,16 @@ export default class ColumnSettings extends React.PureComponent {
           <ClearColumnButton onClick={onClear} />
         </div>
 
+        <div role='group' aria-labelledby='notifications-unread-markers'>
+          <span id='notifications-unread-markers' className='column-settings__section'>
+            <FormattedMessage id='notifications.column_settings.unread_markers.category' defaultMessage='Unread notification markers' />
+          </span>
+
+          <div className='column-settings__row'>
+            <SettingToggle id='unread-notification-markers' prefix='notifications' settings={settings} settingPath={['showUnread']} onChange={onChange} label={filterShowStr} />
+          </div>
+        </div>
+
         <div role='group' aria-labelledby='notifications-filter-bar'>
           <span id='notifications-filter-bar' className='column-settings__section'>
             <FormattedMessage id='notifications.column_settings.filter_bar.category' defaultMessage='Quick filter bar' />

app/javascript/mastodon/features/notifications/index.js

@@ -60,8 +60,8 @@ const mapStateToProps = state => ({
   isUnread: state.getIn(['notifications', 'unread']) > 0 || state.getIn(['notifications', 'pendingItems']).size > 0,
   hasMore: state.getIn(['notifications', 'hasMore']),
   numPending: state.getIn(['notifications', 'pendingItems'], ImmutableList()).size,
-  lastReadId: state.getIn(['notifications', 'readMarkerId']),
-  canMarkAsRead: state.getIn(['notifications', 'readMarkerId']) !== '0' && getNotifications(state).some(item => item !== null && compareId(item.get('id'), state.getIn(['notifications', 'readMarkerId'])) > 0),
+  lastReadId: state.getIn(['settings', 'notifications', 'showUnread']) ? state.getIn(['notifications', 'readMarkerId']) : '0',
+  canMarkAsRead: state.getIn(['settings', 'notifications', 'showUnread']) && state.getIn(['notifications', 'readMarkerId']) !== '0' && getNotifications(state).some(item => item !== null && compareId(item.get('id'), state.getIn(['notifications', 'readMarkerId'])) > 0),
   needsNotificationPermission: state.getIn(['settings', 'notifications', 'alerts']).includes(true) && state.getIn(['notifications', 'browserSupport']) && state.getIn(['notifications', 'browserPermission']) === 'default' && !state.getIn(['settings', 'notifications', 'dismissPermissionBanner']),
 });
 

app/javascript/mastodon/reducers/settings.js

@@ -45,6 +45,7 @@ const initialState = ImmutableMap({
     }),
 
     dismissPermissionBanner: false,
+    showUnread: true,
 
     shows: ImmutableMap({
       follow: true,

app/lib/entity_cache.rb

@@ -16,7 +16,9 @@ class EntityCache
   end
 
   def emoji(shortcodes, domain)
-    shortcodes   = Array(shortcodes)
+    shortcodes = Array(shortcodes)
+    return [] if shortcodes.empty?
+
     cached       = Rails.cache.read_multi(*shortcodes.map { |shortcode| to_key(:emoji, shortcode, domain) })
     uncached_ids = []
 

app/lib/formatter.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'singleton'
-require_relative './sanitize_config'
 
 class HTMLRenderer < Redcarpet::Render::HTML
   def block_code(code, language)
@@ -223,9 +222,9 @@ class Formatter
           original_url, static_url = emoji
           replacement = begin
             if animate
-              "<img draggable=\"false\" class=\"emojione\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(original_url)}\" />"
+              image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:")
             else
-              "<img draggable=\"false\" class=\"emojione custom-emoji\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(static_url)}\" data-original=\"#{original_url}\" data-static=\"#{static_url}\" />"
+              image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url })
             end
           end
           before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''

app/mailers/notification_mailer.rb

@@ -4,7 +4,7 @@ class NotificationMailer < ApplicationMailer
   helper :accounts
   helper :statuses
 
-  add_template_helper RoutingHelper
+  helper RoutingHelper
 
   def mention(recipient, notification)
     @me     = recipient

app/mailers/user_mailer.rb

@@ -8,7 +8,7 @@ class UserMailer < Devise::Mailer
   helper :instance
   helper :statuses
 
-  add_template_helper RoutingHelper
+  helper RoutingHelper
 
   def confirmation_instructions(user, token, **)
     @resource = user

app/models/account_stat.rb

@@ -18,46 +18,4 @@ class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
   update_index('accounts#account', :account)
-
-  def increment_count!(key)
-    update(attributes_for_increment(key))
-  rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique
-    begin
-      reload_with_id
-    rescue ActiveRecord::RecordNotFound
-      return
-    end
-
-    retry
-  end
-
-  def decrement_count!(key)
-    update(attributes_for_decrement(key))
-  rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique
-    begin
-      reload_with_id
-    rescue ActiveRecord::RecordNotFound
-      return
-    end
-
-    retry
-  end
-
-  private
-
-  def attributes_for_increment(key)
-    attrs = { key => public_send(key) + 1 }
-    attrs[:last_status_at] = Time.now.utc if key == :statuses_count
-    attrs
-  end
-
-  def attributes_for_decrement(key)
-    attrs = { key => [public_send(key) - 1, 0].max }
-    attrs
-  end
-
-  def reload_with_id
-    self.id = self.class.find_by!(account: account).id if new_record?
-    reload
-  end
 end

app/models/concerns/account_counters.rb

@@ -3,6 +3,8 @@
 module AccountCounters
   extend ActiveSupport::Concern
 
+  ALLOWED_COUNTER_KEYS = %i(statuses_count following_count followers_count).freeze
+
   included do
     has_one :account_stat, inverse_of: :account
     after_save :save_account_stat
@@ -14,11 +16,65 @@ module AccountCounters
            :following_count=,
            :followers_count,
            :followers_count=,
-           :increment_count!,
-           :decrement_count!,
            :last_status_at,
            to: :account_stat
 
+  # @param [Symbol] key
+  def increment_count!(key)
+    update_count!(key, 1)
+  end
+
+  # @param [Symbol] key
+  def decrement_count!(key)
+    update_count!(key, -1)
+  end
+
+  # @param [Symbol] key
+  # @param [Integer] value
+  def update_count!(key, value)
+    raise ArgumentError, "Invalid key #{key}" unless ALLOWED_COUNTER_KEYS.include?(key)
+    raise ArgumentError, 'Do not call update_count! on dirty objects' if association(:account_stat).loaded? && account_stat&.changed? && account_stat.changed_attribute_names_to_save == %w(id)
+
+    value = value.to_i
+    default_value = value.positive? ? value : 0
+
+    # We do an upsert using manually written SQL, as Rails' upsert method does
+    # not seem to support writing expressions in the UPDATE clause, but only
+    # re-insert the provided values instead.
+    # Even ARel seem to be missing proper handling of upserts.
+    sql = if value.positive? && key == :statuses_count
+            <<-SQL.squish
+              INSERT INTO account_stats(account_id, #{key}, created_at, updated_at, last_status_at)
+                VALUES (:account_id, :default_value, now(), now(), now())
+              ON CONFLICT (account_id) DO UPDATE
+              SET #{key} = account_stats.#{key} + :value,
+                  last_status_at = now(),
+                  lock_version = account_stats.lock_version + 1,
+                  updated_at = now()
+              RETURNING id;
+            SQL
+          else
+            <<-SQL.squish
+              INSERT INTO account_stats(account_id, #{key}, created_at, updated_at)
+                VALUES (:account_id, :default_value, now(), now())
+              ON CONFLICT (account_id) DO UPDATE
+              SET #{key} = account_stats.#{key} + :value,
+                  lock_version = account_stats.lock_version + 1,
+                  updated_at = now()
+              RETURNING id;
+            SQL
+          end
+
+    sql = AccountStat.sanitize_sql([sql, account_id: id, default_value: default_value, value: value])
+    account_stat_id = AccountStat.connection.exec_query(sql)[0]['id']
+
+    # Reload account_stat if it was loaded, taking into account newly-created unsaved records
+    if association(:account_stat).loaded?
+      account_stat.id = account_stat_id if account_stat.new_record?
+      account_stat.reload
+    end
+  end
+
   def account_stat
     super || build_account_stat
   end

app/models/notification.rb

@@ -49,12 +49,12 @@ class Notification < ApplicationRecord
   belongs_to :from_account, class_name: 'Account', optional: true
   belongs_to :activity, polymorphic: true, optional: true
 
-  belongs_to :mention,        foreign_type: 'Mention',       foreign_key: 'activity_id', optional: true
-  belongs_to :status,         foreign_type: 'Status',        foreign_key: 'activity_id', optional: true
-  belongs_to :follow,         foreign_type: 'Follow',        foreign_key: 'activity_id', optional: true
-  belongs_to :follow_request, foreign_type: 'FollowRequest', foreign_key: 'activity_id', optional: true
-  belongs_to :favourite,      foreign_type: 'Favourite',     foreign_key: 'activity_id', optional: true
-  belongs_to :poll,           foreign_type: 'Poll',          foreign_key: 'activity_id', optional: true
+  belongs_to :mention,        foreign_key: 'activity_id', optional: true
+  belongs_to :status,         foreign_key: 'activity_id', optional: true
+  belongs_to :follow,         foreign_key: 'activity_id', optional: true
+  belongs_to :follow_request, foreign_key: 'activity_id', optional: true
+  belongs_to :favourite,      foreign_key: 'activity_id', optional: true
+  belongs_to :poll,           foreign_key: 'activity_id', optional: true
 
   validates :type, inclusion: { in: TYPES }
 

app/validators/url_validator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class UrlValidator < ActiveModel::EachValidator
+class URLValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
     record.errors.add(attribute, I18n.t('applications.invalid_url')) unless compliant?(value)
   end

config/application.rb

@@ -6,8 +6,9 @@ require 'rails/all'
 # you've limited to :test, :development, or :production.
 Bundler.require(*Rails.groups)
 
-require_relative '../app/lib/exceptions'
+require_relative '../lib/exceptions'
 require_relative '../lib/enumerable'
+require_relative '../lib/sanitize_ext/sanitize_config'
 require_relative '../lib/redis/namespace_extensions'
 require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
@@ -27,6 +28,7 @@ require_relative '../lib/webpacker/manifest_extensions'
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/action_dispatch/cookie_jar_extensions'
 require_relative '../lib/rails/engine_extensions'
+require_relative '../lib/active_record/database_tasks_extensions'
 
 Dotenv::Railtie.load
 

config/environments/development.rb

@@ -17,7 +17,7 @@ Rails.application.configure do
   if Rails.root.join('tmp/caching-dev.txt').exist?
     config.action_controller.perform_caching = true
 
-    config.cache_store = :redis_store, ENV['REDIS_URL'], REDIS_CACHE_PARAMS
+    config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS
 
     config.public_file_server.headers = {
       'Cache-Control' => "public, max-age=#{2.days.to_i}",

config/environments/production.rb

@@ -52,7 +52,7 @@ Rails.application.configure do
   config.log_tags = [:request_id]
 
   # Use a different cache store in production.
-  config.cache_store = :redis_store, ENV['CACHE_REDIS_URL'], REDIS_CACHE_PARAMS
+  config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS
 
   # Ignore bad email addresses and do not raise email delivery errors.
   # Set this to true and configure the email server for immediate delivery to raise delivery errors.

config/initializers/inflections.rb

@@ -20,6 +20,10 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
   inflect.acronym 'JsonLd'
   inflect.acronym 'NodeInfo'
   inflect.acronym 'Ed25519'
+  inflect.acronym 'TOC'
+  inflect.acronym 'RSS'
+  inflect.acronym 'REST'
+  inflect.acronym 'URL'
 
   inflect.singular 'data', 'data'
 end

config/initializers/sidekiq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 namespace    = ENV.fetch('REDIS_NAMESPACE') { nil }
-redis_params = { url: ENV['REDIS_URL'] }
+redis_params = { url: ENV['REDIS_URL'], driver: :hiredis }
 
 if namespace
   redis_params[:namespace] = namespace

db/migrate/20160223165723_add_url_to_statuses.rb

@@ -1,4 +1,4 @@
-class AddUrlToStatuses < ActiveRecord::Migration[4.2]
+class AddURLToStatuses < ActiveRecord::Migration[4.2]
   def change
     add_column :statuses, :url, :string, null: true, default: nil
   end

db/migrate/20160223165855_add_url_to_accounts.rb

@@ -1,4 +1,4 @@
-class AddUrlToAccounts < ActiveRecord::Migration[4.2]
+class AddURLToAccounts < ActiveRecord::Migration[4.2]
   def change
     add_column :accounts, :url, :string, null: true, default: nil
   end

db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb

@@ -1,4 +1,4 @@
-class AddAvatarRemoteUrlToAccounts < ActiveRecord::Migration[4.2]
+class AddAvatarRemoteURLToAccounts < ActiveRecord::Migration[4.2]
   def change
     add_column :accounts, :avatar_remote_url, :string, null: true, default: nil
   end

db/migrate/20161006213403_rails_settings_migration.rb

@@ -7,12 +7,12 @@ end
 class RailsSettingsMigration < MIGRATION_BASE_CLASS
   def self.up
     create_table :settings do |t|
-      t.string     :var, :null => false
+      t.string     :var, null: false
       t.text       :value
-      t.references :target, :null => false, :polymorphic => true
-      t.timestamps :null => true
+      t.references :target, null: false, polymorphic: true, index: { name: 'index_settings_on_target_type_and_target_id' }
+      t.timestamps null: true
     end
-    add_index :settings, [ :target_type, :target_id, :var ], :unique => true
+    add_index :settings, [ :target_type, :target_id, :var ], unique: true
   end
 
   def self.down

db/migrate/20170318214217_add_header_remote_url_to_accounts.rb

@@ -1,4 +1,4 @@
-class AddHeaderRemoteUrlToAccounts < ActiveRecord::Migration[5.0]
+class AddHeaderRemoteURLToAccounts < ActiveRecord::Migration[5.0]
   def change
     add_column :accounts, :header_remote_url, :string, null: false, default: ''
   end

db/migrate/20170920024819_status_ids_to_timestamp_ids.rb

@@ -1,7 +1,7 @@
 class StatusIdsToTimestampIds < ActiveRecord::Migration[5.1]
   def up
     # Prepare the function we will use to generate IDs.
-    Rake::Task['db:define_timestamp_id'].execute
+    Mastodon::Snowflake.define_timestamp_id
 
     # Set up the statuses.id column to use our timestamp-based IDs.
     ActiveRecord::Base.connection.execute(<<~SQL)
@@ -11,7 +11,7 @@ class StatusIdsToTimestampIds < ActiveRecord::Migration[5.1]
     SQL
 
     # Make sure we have a sequence to use.
-    Rake::Task['db:ensure_id_sequences_exist'].execute
+    Mastodon::Snowflake.ensure_id_sequences_exist
   end
 
   def down

db/migrate/20171119172437_create_admin_action_logs.rb

@@ -3,7 +3,7 @@ class CreateAdminActionLogs < ActiveRecord::Migration[5.1]
     create_table :admin_action_logs do |t|
       t.belongs_to :account, foreign_key: { on_delete: :cascade }
       t.string :action, null: false, default: ''
-      t.references :target, polymorphic: true
+      t.references :target, polymorphic: true, index: { name: 'index_admin_action_logs_on_target_type_and_target_id' }
       t.text :recorded_changes, null: false, default: ''
 
       t.timestamps

db/migrate/20171130000000_add_embed_url_to_preview_cards.rb

@@ -1,6 +1,6 @@
 require Rails.root.join('lib', 'mastodon', 'migration_helpers')
 
-class AddEmbedUrlToPreviewCards < ActiveRecord::Migration[5.1]
+class AddEmbedURLToPreviewCards < ActiveRecord::Migration[5.1]
   include Mastodon::MigrationHelpers
 
   disable_ddl_transaction!

db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb

@@ -1,4 +1,4 @@
-class AddFeaturedCollectionUrlToAccounts < ActiveRecord::Migration[5.1]
+class AddFeaturedCollectionURLToAccounts < ActiveRecord::Migration[5.1]
   def change
     add_column :accounts, :featured_collection_url, :string
   end

db/migrate/20180528141303_fix_accounts_unique_index.rb

@@ -37,7 +37,7 @@ class FixAccountsUniqueIndex < ActiveRecord::Migration[5.2]
       end
     end
 
-    duplicates = Account.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM accounts GROUP BY lower(username), lower(domain) HAVING count(*) > 1').to_hash
+    duplicates = Account.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM accounts GROUP BY lower(username), lower(domain) HAVING count(*) > 1').to_ary
 
     duplicates.each do |row|
       deduplicate_account!(row['ids'].split(','))

db/migrate/20181024224956_migrate_account_conversations.rb

@@ -17,8 +17,8 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
     belongs_to :account, optional: true
     belongs_to :activity, polymorphic: true, optional: true
 
-    belongs_to :status,         foreign_type: 'Status',        foreign_key: 'activity_id', optional: true
-    belongs_to :mention,        foreign_type: 'Mention',       foreign_key: 'activity_id', optional: true
+    belongs_to :status,  foreign_key: 'activity_id', optional: true
+    belongs_to :mention, foreign_key: 'activity_id', optional: true
 
     def target_status
       mention&.status

db/migrate/20181207011115_downcase_custom_emoji_domains.rb

@@ -2,7 +2,7 @@ class DowncaseCustomEmojiDomains < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
-    duplicates = CustomEmoji.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM custom_emojis GROUP BY shortcode, lower(domain) HAVING count(*) > 1').to_hash
+    duplicates = CustomEmoji.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM custom_emojis GROUP BY shortcode, lower(domain) HAVING count(*) > 1').to_ary
 
     duplicates.each do |row|
       CustomEmoji.where(id: row['ids'].split(',')[0...-1]).destroy_all

db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

@@ -2,7 +2,7 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
-    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row|
+    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_ary.each do |row|
       canonical_tag_id  = row['ids'].split(',').first
       redundant_tag_ids = row['ids'].split(',')[1..-1]
 

db/migrate/20200529214050_add_devices_url_to_accounts.rb

@@ -1,4 +1,4 @@
-class AddDevicesUrlToAccounts < ActiveRecord::Migration[5.2]
+class AddDevicesURLToAccounts < ActiveRecord::Migration[5.2]
   def change
     add_column :accounts, :devices_url, :string
   end

lib/active_record/database_tasks_extensions.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative '../mastodon/snowflake'
+
+module ActiveRecord
+  module Tasks
+    module DatabaseTasks
+      original_load_schema = instance_method(:load_schema)
+
+      define_method(:load_schema) do |db_config, *args|
+        ActiveRecord::Base.establish_connection(db_config)
+        Mastodon::Snowflake.define_timestamp_id
+
+        original_load_schema.bind(self).call(db_config, *args)
+
+        Mastodon::Snowflake.ensure_id_sequences_exist
+      end
+    end
+  end
+end

lib/mastodon/maintenance_cli.rb

@@ -14,7 +14,7 @@ module Mastodon
     end
 
     MIN_SUPPORTED_VERSION = 2019_10_01_213028
-    MAX_SUPPORTED_VERSION = 2020_12_18_054746
+    MAX_SUPPORTED_VERSION = 2021_03_08_133107
 
     # Stubs to enjoy ActiveRecord queries while not depending on a particular
     # version of the code/database
@@ -142,7 +142,6 @@ module Mastodon
       @prompt.warn 'Please make sure to stop Mastodon and have a backup.'
       exit(1) unless @prompt.yes?('Continue?')
 
-      deduplicate_accounts!
       deduplicate_users!
       deduplicate_account_domain_blocks!
       deduplicate_account_identity_proofs!
@@ -157,6 +156,7 @@ module Mastodon
       deduplicate_media_attachments!
       deduplicate_preview_cards!
       deduplicate_statuses!
+      deduplicate_accounts!
       deduplicate_tags!
       deduplicate_webauthn_credentials!
 

lib/mastodon/migration_helpers.rb

@@ -41,42 +41,18 @@
 
 module Mastodon
   module MigrationHelpers
-    # Stub for Database.postgresql? from GitLab
-    def self.postgresql?
-      ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero?
-    end
-
-    # Stub for Database.mysql? from GitLab
-    def self.mysql?
-      ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('mysql2').zero?
-    end
-
     # Model that can be used for querying permissions of a SQL user.
     class Grant < ActiveRecord::Base
-      self.table_name =
-        if Mastodon::MigrationHelpers.postgresql?
-          'information_schema.role_table_grants'
-        else
-          'mysql.user'
-        end
+      self.table_name = 'information_schema.role_table_grants'
 
       def self.scope_to_current_user
-        if Mastodon::MigrationHelpers.postgresql?
-          where('grantee = user')
-        else
-          where("CONCAT(User, '@', Host) = current_user()")
-        end
+        where('grantee = user')
       end
 
       # Returns true if the current user can create and execute triggers on the
       # given table.
       def self.create_and_execute_trigger?(table)
-        priv =
-          if Mastodon::MigrationHelpers.postgresql?
-            where(privilege_type: 'TRIGGER', table_name: table)
-          else
-            where(Trigger_priv: 'Y')
-          end
+        priv = where(privilege_type: 'TRIGGER', table_name: table)
 
         priv.scope_to_current_user.any?
       end
@@ -141,10 +117,8 @@ module Mastodon
           'in the body of your migration class'
       end
 
-      if MigrationHelpers.postgresql?
-        options = options.merge({ algorithm: :concurrently })
-        disable_statement_timeout
-      end
+      options = options.merge({ algorithm: :concurrently })
+      disable_statement_timeout
 
       add_index(table_name, column_name, options)
     end
@@ -199,8 +173,6 @@ module Mastodon
 
     # Only available on Postgresql >= 9.2
     def supports_drop_index_concurrently?
-      return false unless MigrationHelpers.postgresql?
-
       version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
 
       version >= 90200
@@ -226,13 +198,7 @@ module Mastodon
       # While MySQL does allow disabling of foreign keys it has no equivalent
       # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
       # back to the normal foreign key procedure.
-      if MigrationHelpers.mysql?
-        return add_foreign_key(source, target,
-                               column: column,
-                               on_delete: on_delete)
-      else
-        on_delete = 'SET NULL' if on_delete == :nullify
-      end
+      on_delete = 'SET NULL' if on_delete == :nullify
 
       disable_statement_timeout
 
@@ -270,7 +236,7 @@ module Mastodon
     # the database. Disable the session's statement timeout to ensure
     # migrations don't get killed prematurely. (PostgreSQL only)
     def disable_statement_timeout
-      execute('SET statement_timeout TO 0') if MigrationHelpers.postgresql?
+      execute('SET statement_timeout TO 0')
     end
 
     # Updates the value of a column in batches.
@@ -319,7 +285,7 @@ module Mastodon
         count_arel = table.project(Arel.star.count.as('count'))
         count_arel = yield table, count_arel if block_given?
 
-        total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
+        total = exec_query(count_arel.to_sql).to_ary.first['count'].to_i
 
         return if total == 0
       end
@@ -335,7 +301,7 @@ module Mastodon
 
       start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
       start_arel = yield table, start_arel if block_given?
-      first_row = exec_query(start_arel.to_sql).to_hash.first
+      first_row = exec_query(start_arel.to_sql).to_ary.first
       # In case there are no rows but we didn't catch it in the estimated size:
       return unless first_row
       start_id = first_row['id'].to_i
@@ -356,7 +322,7 @@ module Mastodon
             .skip(batch_size)
 
           stop_arel = yield table, stop_arel if block_given?
-          stop_row = exec_query(stop_arel.to_sql).to_hash.first
+          stop_row = exec_query(stop_arel.to_sql).to_ary.first
 
           update_arel = Arel::UpdateManager.new
             .table(table)
@@ -487,11 +453,7 @@ module Mastodon
       # If we were in the middle of update_column_in_batches, we should remove
       # the old column and start over, as we have no idea where we were.
       if column_for(table, new)
-        if MigrationHelpers.postgresql?
-          remove_rename_triggers_for_postgresql(table, trigger_name)
-        else
-          remove_rename_triggers_for_mysql(trigger_name)
-        end
+        remove_rename_triggers_for_postgresql(table, trigger_name)
 
         remove_column(table, new)
       end
@@ -521,13 +483,8 @@ module Mastodon
       quoted_old = quote_column_name(old)
       quoted_new = quote_column_name(new)
 
-      if MigrationHelpers.postgresql?
-        install_rename_triggers_for_postgresql(trigger_name, quoted_table,
-                                               quoted_old, quoted_new)
-      else
-        install_rename_triggers_for_mysql(trigger_name, quoted_table,
-                                          quoted_old, quoted_new)
-      end
+      install_rename_triggers_for_postgresql(trigger_name, quoted_table,
+                                             quoted_old, quoted_new)
 
       update_column_in_batches(table, new, Arel::Table.new(table)[old])
 
@@ -685,11 +642,7 @@ module Mastodon
 
       check_trigger_permissions!(table)
 
-      if MigrationHelpers.postgresql?
-        remove_rename_triggers_for_postgresql(table, trigger_name)
-      else
-        remove_rename_triggers_for_mysql(trigger_name)
-      end
+      remove_rename_triggers_for_postgresql(table, trigger_name)
 
       remove_column(table, old)
     end
@@ -844,18 +797,9 @@ module Mastodon
       quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s)
       quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s)
 
-      if MigrationHelpers.mysql?
-        locate = Arel::Nodes::NamedFunction
-          .new('locate', [quoted_pattern, column])
-        insert_in_place = Arel::Nodes::NamedFunction
-          .new('insert', [column, locate, pattern.size, quoted_replacement])
-
-        Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql)
-      else
-        replace = Arel::Nodes::NamedFunction
-          .new("regexp_replace", [column, quoted_pattern, quoted_replacement])
-        Arel::Nodes::SqlLiteral.new(replace.to_sql)
-      end
+      replace = Arel::Nodes::NamedFunction
+        .new("regexp_replace", [column, quoted_pattern, quoted_replacement])
+      Arel::Nodes::SqlLiteral.new(replace.to_sql)
     end
 
     def remove_foreign_key_without_error(*args)

lib/mastodon/redis_config.rb

@@ -27,6 +27,8 @@ namespace       = ENV.fetch('REDIS_NAMESPACE', nil)
 cache_namespace = namespace ? namespace + '_cache' : 'cache'
 
 REDIS_CACHE_PARAMS = {
+  driver: :hiredis,
+  url: ENV['REDIS_URL'],
   expires_in: 10.minutes,
   namespace: cache_namespace,
 }.freeze

lib/tasks/db.rake

@@ -1,36 +1,5 @@
 # frozen_string_literal: true
 
-require_relative '../mastodon/snowflake'
-
-def each_schema_load_environment
-  # If we're in development, also run this for the test environment.
-  # This is a somewhat hacky way to do this, so here's why:
-  # 1. We have to define this before we load the schema, or we won't
-  #    have a timestamp_id function when we get to it in the schema.
-  # 2. db:setup calls db:schema:load_if_ruby, which calls
-  #    db:schema:load, which we define above as having a prerequisite
-  #    of this task.
-  # 3. db:schema:load ends up running
-  #    ActiveRecord::Tasks::DatabaseTasks.load_schema_current, which
-  #    calls a private method `each_current_configuration`, which
-  #    explicitly also does the loading for the `test` environment
-  #    if the current environment is `development`, so we end up
-  #    needing to do the same, and we can't even use the same method
-  #    to do it.
-
-  if Rails.env.development?
-    test_conf = ActiveRecord::Base.configurations['test']
-
-    if test_conf['database']&.present?
-      ActiveRecord::Base.establish_connection(:test)
-      yield
-      ActiveRecord::Base.establish_connection(Rails.env.to_sym)
-    end
-  end
-
-  yield
-end
-
 namespace :db do
   namespace :migrate do
     desc 'Setup the db or migrate depending on state of db'
@@ -50,7 +19,7 @@ namespace :db do
 
   task :post_migration_hook do
     at_exit do
-      unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate'])
+      unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
         warn <<~WARNING
           Your database collation is susceptible to index corruption.
             (This warning does not indicate that index corruption has occured and can be ignored)
@@ -60,30 +29,11 @@ namespace :db do
     end
   end
 
-  Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
-
-  # Before we load the schema, define the timestamp_id function.
-  # Idiomatically, we might do this in a migration, but then it
-  # wouldn't end up in schema.rb, so we'd need to figure out a way to
-  # get it in before doing db:setup as well. This is simpler, and
-  # ensures it's always in place.
-  Rake::Task['db:schema:load'].enhance ['db:define_timestamp_id']
-
-  # After we load the schema, make sure we have sequences for each
-  # table using timestamp IDs.
-  Rake::Task['db:schema:load'].enhance do
-    Rake::Task['db:ensure_id_sequences_exist'].invoke
-  end
-
-  task :define_timestamp_id do
-    each_schema_load_environment do
-      Mastodon::Snowflake.define_timestamp_id
-    end
+  task :pre_migration_check do
+    version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+    abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500
   end
 
-  task :ensure_id_sequences_exist do
-    each_schema_load_environment do
-      Mastodon::Snowflake.ensure_id_sequences_exist
-    end
-  end
+  Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
+  Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
 end

package.json

@@ -60,14 +60,14 @@
   },
   "private": true,
   "dependencies": {
-    "@babel/core": "^7.13.8",
+    "@babel/core": "^7.13.10",
     "@babel/plugin-proposal-class-properties": "^7.8.3",
     "@babel/plugin-proposal-decorators": "^7.13.5",
     "@babel/plugin-transform-react-inline-elements": "^7.12.13",
-    "@babel/plugin-transform-runtime": "^7.13.9",
-    "@babel/preset-env": "^7.13.9",
+    "@babel/plugin-transform-runtime": "^7.13.10",
+    "@babel/preset-env": "^7.13.10",
     "@babel/preset-react": "^7.12.13",
-    "@babel/runtime": "^7.13.9",
+    "@babel/runtime": "^7.13.10",
     "@clusterws/cws": "^3.0.0",
     "@gamestdio/websocket": "^0.3.2",
     "@github/webauthn-json": "^0.5.7",
@@ -88,7 +88,7 @@
     "color-blend": "^3.0.1",
     "compression-webpack-plugin": "^6.1.1",
     "cross-env": "^7.0.3",
-    "css-loader": "^5.1.1",
+    "css-loader": "^5.1.2",
     "cssnano": "^4.1.10",
     "detect-passive-events": "^2.0.3",
     "dotenv": "^8.2.0",
@@ -146,7 +146,7 @@
     "react-sparklines": "^1.7.0",
     "react-swipeable-views": "^0.13.9",
     "react-textarea-autosize": "^8.3.2",
-    "react-toggle": "^4.1.1",
+    "react-toggle": "^4.1.2",
     "redis": "^3.0.2",
     "redux": "^4.0.5",
     "redux-immutable": "^4.0.0",
@@ -179,7 +179,7 @@
     "@testing-library/react": "^11.2.5",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.6.3",
-    "eslint": "^7.21.0",
+    "eslint": "^7.22.0",
     "eslint-plugin-import": "~2.22.1",
     "eslint-plugin-jsx-a11y": "~6.4.1",
     "eslint-plugin-promise": "~4.3.1",

spec/controllers/accounts_controller_spec.rb

@@ -370,7 +370,7 @@ RSpec.describe AccountsController, type: :controller do
         end
 
         it 'returns application/activity+json' do
-          expect(response.content_type).to eq 'application/activity+json'
+          expect(response.media_type).to eq 'application/activity+json'
         end
 
         it_behaves_like 'cachable response'
@@ -402,7 +402,7 @@ RSpec.describe AccountsController, type: :controller do
         end
 
         it 'returns application/activity+json' do
-          expect(response.content_type).to eq 'application/activity+json'
+          expect(response.media_type).to eq 'application/activity+json'
         end
 
         it 'returns public Cache-Control header' do