Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `Gemfile.lock`:
  Not a real conflict, upstream-updated dependency (redis) textually too
  close to glitch-soc-only dependecy.
  Updated redis gem like upstream did.
master
Claire 2 years ago
commit 8ec4be4233
  1. 12
      Gemfile
  2. 55
      Gemfile.lock
  3. 35
      app/controllers/statuses_cleanup_controller.rb
  4. 1
      app/javascript/mastodon/features/compose/containers/navigation_container.js
  5. 1
      app/javascript/mastodon/features/compose/index.js
  6. 9
      app/javascript/mastodon/features/ui/components/confirmation_modal.js
  7. 1
      app/javascript/mastodon/features/ui/components/link_footer.js
  8. 4
      app/javascript/styles/mastodon/components.scss
  9. 6
      app/lib/activitypub/activity/create.rb
  10. 171
      app/models/account_statuses_cleanup_policy.rb
  11. 8
      app/models/bookmark.rb
  12. 3
      app/models/concerns/account_associations.rb
  13. 3
      app/models/concerns/account_interactions.rb
  14. 7
      app/models/favourite.rb
  15. 8
      app/models/status_pin.rb
  16. 27
      app/services/account_statuses_cleanup_service.rb
  17. 2
      app/services/delete_account_service.rb
  18. 45
      app/views/statuses_cleanup/show.html.haml
  19. 4
      app/workers/move_worker.rb
  20. 96
      app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
  21. 35
      config/locales/en.yml
  22. 1
      config/navigation.rb
  23. 1
      config/routes.rb
  24. 4
      config/sidekiq.yml
  25. 20
      db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb
  26. 21
      db/post_migrate/20210808071221_clear_orphaned_account_notes.rb
  27. 20
      db/schema.rb
  28. 7
      lib/mastodon/snowflake.rb
  29. 2
      lib/paperclip/transcoder.rb
  30. 18
      package.json
  31. 27
      spec/controllers/statuses_cleanup_controller_spec.rb
  32. 3
      spec/fabricators/account_statuses_cleanup_policy_fabricator.rb
  33. 546
      spec/models/account_statuses_cleanup_policy_spec.rb
  34. 101
      spec/services/account_statuses_cleanup_service_spec.rb
  35. 5
      spec/services/delete_account_service_spec.rb
  36. 127
      spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb
  37. 372
      yarn.lock

@ -5,7 +5,7 @@ ruby '>= 2.5.0', '< 3.1.0'
gem 'pkg-config', '~> 1.4'
gem 'puma', '~> 5.3'
gem 'puma', '~> 5.4'
gem 'rails', '~> 6.1.4'
gem 'sprockets', '~> 3.7.2'
gem 'thor', '~> 1.1'
@ -17,7 +17,7 @@ gem 'makara', '~> 0.5'
gem 'pghero', '~> 2.8'
gem 'dotenv-rails', '~> 2.7'
gem 'aws-sdk-s3', '~> 1.96', require: false
gem 'aws-sdk-s3', '~> 1.98', require: false
gem 'fog-core', '<= 2.1.0'
gem 'fog-openstack', '~> 0.3', require: false
gem 'paperclip', '~> 6.0'
@ -60,7 +60,7 @@ gem 'idn-ruby', require: 'idn'
gem 'kaminari', '~> 1.2'
gem 'link_header', '~> 0.0'
gem 'mime-types', '~> 3.3.1', require: 'mime/types/columnar'
gem 'nokogiri', '~> 1.11'
gem 'nokogiri', '~> 1.12'
gem 'nsa', '~> 0.2'
gem 'oj', '~> 3.12'
gem 'ox', '~> 2.14'
@ -73,11 +73,11 @@ gem 'rack-attack', '~> 6.5'
gem 'rack-cors', '~> 1.1', require: 'rack/cors'
gem 'rails-i18n', '~> 6.0'
gem 'rails-settings-cached', '~> 0.6'
gem 'redis', '~> 4.3', require: ['redis', 'redis/connection/hiredis']
gem 'redis', '~> 4.4', require: ['redis', 'redis/connection/hiredis']
gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
gem 'rqrcode', '~> 2.0'
gem 'ruby-progressbar', '~> 1.11'
gem 'sanitize', '~> 5.2'
gem 'sanitize', '~> 6.0'
gem 'scenic', '~> 1.5'
gem 'sidekiq', '~> 6.2'
gem 'sidekiq-scheduler', '~> 3.1'
@ -138,7 +138,7 @@ group :development do
gem 'memory_profiler'
gem 'rubocop', '~> 1.18', require: false
gem 'rubocop-rails', '~> 2.11', require: false
gem 'brakeman', '~> 5.0', require: false
gem 'brakeman', '~> 5.1', require: false
gem 'bundler-audit', '~> 0.8', require: false
gem 'capistrano', '~> 3.16'

@ -79,20 +79,20 @@ GEM
encryptor (~> 3.0.0)
awrence (1.1.1)
aws-eventstream (1.1.1)
aws-partitions (1.467.0)
aws-sdk-core (3.114.2)
aws-partitions (1.482.0)
aws-sdk-core (3.119.0)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.239.0)
aws-sigv4 (~> 1.1)
jmespath (~> 1.0)
aws-sdk-kms (1.43.0)
aws-sdk-core (~> 3, >= 3.112.0)
aws-sdk-kms (1.46.0)
aws-sdk-core (~> 3, >= 3.119.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.96.1)
aws-sdk-core (~> 3, >= 3.112.0)
aws-sdk-s3 (1.98.0)
aws-sdk-core (~> 3, >= 3.119.0)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.1)
aws-sigv4 (1.2.3)
aws-sigv4 (1.2.4)
aws-eventstream (~> 1, >= 1.0.2)
bcrypt (3.1.16)
better_errors (2.9.1)
@ -106,7 +106,7 @@ GEM
ffi (~> 1.14)
bootsnap (1.6.0)
msgpack (~> 1.0)
brakeman (5.0.4)
brakeman (5.1.1)
browser (4.2.0)
brpoplpush-redis_script (0.1.2)
concurrent-ruby (~> 1.0, >= 1.0.5)
@ -354,7 +354,7 @@ GEM
nokogiri (~> 1)
rake
mini_mime (1.1.0)
mini_portile2 (2.5.3)
mini_portile2 (2.6.1)
minitest (5.14.4)
msgpack (1.4.2)
multi_json (1.15.0)
@ -364,17 +364,15 @@ GEM
net-ssh (>= 2.6.5, < 7.0.0)
net-ssh (6.1.0)
nio4r (2.5.7)
nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
nokogiri (1.12.2)
mini_portile2 (~> 2.6.1)
racc (~> 1.4)
nokogumbo (2.0.4)
nokogiri (~> 1.8, >= 1.8.4)
nsa (0.2.8)
activesupport (>= 4.2, < 7)
concurrent-ruby (~> 1.0, >= 1.0.2)
sidekiq (>= 3.5)
statsd-ruby (~> 1.4, >= 1.4.0)
oj (3.12.1)
oj (3.12.2)
omniauth (1.9.1)
hashie (>= 3.4.6)
rack (>= 1.6.2, < 3)
@ -428,7 +426,7 @@ GEM
pry-rails (0.3.9)
pry (>= 0.10.4)
public_suffix (4.0.6)
puma (5.3.2)
puma (5.4.0)
nio4r (~> 2.0)
pundit (2.1.0)
activesupport (>= 3.0.0)
@ -486,7 +484,7 @@ GEM
rdf-normalize (0.4.0)
rdf (~> 3.1)
redcarpet (3.5.1)
redis (4.3.1)
redis (4.4.0)
redis-namespace (1.8.1)
redis (>= 3.0.4)
regexp_parser (2.1.1)
@ -524,16 +522,16 @@ GEM
rspec-support (3.10.2)
rspec_junit_formatter (0.4.1)
rspec-core (>= 2, < 4, != 2.12.0)
rubocop (1.18.3)
rubocop (1.18.4)
parallel (~> 1.10)
parser (>= 3.0.0.0)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml
rubocop-ast (>= 1.7.0, < 2.0)
rubocop-ast (>= 1.8.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.7.0)
rubocop-ast (1.8.0)
parser (>= 3.0.1.1)
rubocop-rails (2.11.3)
activesupport (>= 4.2.0)
@ -547,10 +545,9 @@ GEM
fugit (~> 1.1, >= 1.1.6)
safety_net_attestation (0.4.0)
jwt (~> 2.0)
sanitize (5.2.3)
sanitize (6.0.0)
crass (~> 1.0.2)
nokogiri (>= 1.8.0)
nokogumbo (~> 2.0)
nokogiri (>= 1.12.0)
scenic (1.5.4)
activerecord (>= 4.0.0)
railties (>= 4.0.0)
@ -569,7 +566,7 @@ GEM
sidekiq (>= 3)
thwait
tilt (>= 1.4.0)
sidekiq-unique-jobs (7.1.2)
sidekiq-unique-jobs (7.1.5)
brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
concurrent-ruby (~> 1.0, >= 1.0.5)
sidekiq (>= 5.0, < 7.0)
@ -675,12 +672,12 @@ DEPENDENCIES
active_record_query_trace (~> 1.8)
addressable (~> 2.8)
annotate (~> 3.1)
aws-sdk-s3 (~> 1.96)
aws-sdk-s3 (~> 1.98)
better_errors (~> 2.9)
binding_of_caller (~> 1.0)
blurhash (~> 0.1)
bootsnap (~> 1.6.0)
brakeman (~> 5.0)
brakeman (~> 5.1)
browser
bullet (~> 6.1)
bundler-audit (~> 0.8)
@ -732,7 +729,7 @@ DEPENDENCIES
microformats (~> 4.2)
mime-types (~> 3.3.1)
net-ldap (~> 0.17)
nokogiri (~> 1.11)
nokogiri (~> 1.12)
nsa (~> 0.2)
oj (~> 3.12)
omniauth (~> 1.9)
@ -752,7 +749,7 @@ DEPENDENCIES
private_address_check (~> 0.5)
pry-byebug (~> 3.9)
pry-rails (~> 0.3)
puma (~> 5.3)
puma (~> 5.4)
pundit (~> 2.1)
rack (~> 2.2.3)
rack-attack (~> 6.5)
@ -763,7 +760,7 @@ DEPENDENCIES
rails-settings-cached (~> 0.6)
rdf-normalize (~> 0.4)
redcarpet (~> 3.5)
redis (~> 4.3)
redis (~> 4.4)
redis-namespace (~> 1.8)
rqrcode (~> 2.0)
rspec-rails (~> 5.0)
@ -772,7 +769,7 @@ DEPENDENCIES
rubocop (~> 1.18)
rubocop-rails (~> 2.11)
ruby-progressbar (~> 1.11)
sanitize (~> 5.2)
sanitize (~> 6.0)
scenic (~> 1.5)
sidekiq (~> 6.2)
sidekiq-bulk (~> 0.2.0)

@ -0,0 +1,35 @@
# frozen_string_literal: true
class StatusesCleanupController < ApplicationController
layout 'admin'
before_action :authenticate_user!
before_action :set_policy
before_action :set_body_classes
def show; end
def update
if @policy.update(resource_params)
redirect_to statuses_cleanup_path, notice: I18n.t('generic.changes_saved_msg')
else
render action: :show
end
rescue ActionController::ParameterMissing
# Do nothing
end
private
def set_policy
@policy = current_account.statuses_cleanup_policy || current_account.build_statuses_cleanup_policy(enabled: false)
end
def resource_params
params.require(:account_statuses_cleanup_policy).permit(:enabled, :min_status_age, :keep_direct, :keep_pinned, :keep_polls, :keep_media, :keep_self_fav, :keep_self_bookmark, :min_favs, :min_reblogs)
end
def set_body_classes
@body_classes = 'admin'
end
end

@ -21,6 +21,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
dispatch(openModal('CONFIRM', {
message: intl.formatMessage(messages.logoutMessage),
confirm: intl.formatMessage(messages.logoutConfirm),
closeWhenConfirm: false,
onConfirm: () => logOut(),
}));
},

@ -74,6 +74,7 @@ class Compose extends React.PureComponent {
dispatch(openModal('CONFIRM', {
message: intl.formatMessage(messages.logoutMessage),
confirm: intl.formatMessage(messages.logoutConfirm),
closeWhenConfirm: false,
onConfirm: () => logOut(),
}));

@ -13,15 +13,22 @@ class ConfirmationModal extends React.PureComponent {
onConfirm: PropTypes.func.isRequired,
secondary: PropTypes.string,
onSecondary: PropTypes.func,
closeWhenConfirm: PropTypes.bool,
intl: PropTypes.object.isRequired,
};
static defaultProps = {
closeWhenConfirm: true,
};
componentDidMount() {
this.button.focus();
}
handleClick = () => {
this.props.onClose();
if (this.props.closeWhenConfirm) {
this.props.onClose();
}
this.props.onConfirm();
}

@ -17,6 +17,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
dispatch(openModal('CONFIRM', {
message: intl.formatMessage(messages.logoutMessage),
confirm: intl.formatMessage(messages.logoutConfirm),
closeWhenConfirm: false,
onConfirm: () => logOut(),
}));
},

@ -3022,13 +3022,13 @@ a.account__display-name {
}
@media screen and (max-height: 810px) {
.trends__item:nth-child(3) {
.trends__item:nth-of-type(3) {
display: none;
}
}
@media screen and (max-height: 720px) {
.trends__item:nth-child(2) {
.trends__item:nth-of-type(2) {
display: none;
}
}

@ -452,10 +452,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
end
def supported_blurhash?(blurhash)
components = blurhash.blank? ? nil : Blurhash.components(blurhash)
components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash)
components.present? && components.none? { |comp| comp > 5 }
end
def blurhash_valid_chars?(blurhash)
/^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
end
def skip_download?
return @skip_download if defined?(@skip_download)

@ -0,0 +1,171 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: account_statuses_cleanup_policies
#
# id :bigint not null, primary key
# account_id :bigint not null
# enabled :boolean default(TRUE), not null
# min_status_age :integer default(1209600), not null
# keep_direct :boolean default(TRUE), not null
# keep_pinned :boolean default(TRUE), not null
# keep_polls :boolean default(FALSE), not null
# keep_media :boolean default(FALSE), not null
# keep_self_fav :boolean default(TRUE), not null
# keep_self_bookmark :boolean default(TRUE), not null
# min_favs :integer
# min_reblogs :integer
# created_at :datetime not null
# updated_at :datetime not null
#
class AccountStatusesCleanupPolicy < ApplicationRecord
include Redisable
ALLOWED_MIN_STATUS_AGE = [
2.weeks.seconds,
1.month.seconds,
2.months.seconds,
3.months.seconds,
6.months.seconds,
1.year.seconds,
2.years.seconds,
].freeze
EXCEPTION_BOOLS = %w(keep_direct keep_pinned keep_polls keep_media keep_self_fav keep_self_bookmark).freeze
EXCEPTION_THRESHOLDS = %w(min_favs min_reblogs).freeze
# Depending on the cleanup policy, the query to discover the next
# statuses to delete my get expensive if the account has a lot of old
# statuses otherwise excluded from deletion by the other exceptions.
#
# Therefore, `EARLY_SEARCH_CUTOFF` is meant to be the maximum number of
# old statuses to be considered for deletion prior to checking exceptions.
#
# This is used in `compute_cutoff_id` to provide a `max_id` to
# `statuses_to_delete`.
EARLY_SEARCH_CUTOFF = 5_000
belongs_to :account
validates :min_status_age, inclusion: { in: ALLOWED_MIN_STATUS_AGE }
validates :min_favs, numericality: { greater_than_or_equal_to: 1, allow_nil: true }
validates :min_reblogs, numericality: { greater_than_or_equal_to: 1, allow_nil: true }
validate :validate_local_account
before_save :update_last_inspected
def statuses_to_delete(limit = 50, max_id = nil, min_id = nil)
scope = account.statuses
scope.merge!(old_enough_scope(max_id))
scope = scope.where(Status.arel_table[:id].gteq(min_id)) if min_id.present?
scope.merge!(without_popular_scope) unless min_favs.nil? && min_reblogs.nil?
scope.merge!(without_direct_scope) if keep_direct?
scope.merge!(without_pinned_scope) if keep_pinned?
scope.merge!(without_poll_scope) if keep_polls?
scope.merge!(without_media_scope) if keep_media?
scope.merge!(without_self_fav_scope) if keep_self_fav?
scope.merge!(without_self_bookmark_scope) if keep_self_bookmark?
scope.reorder(id: :asc).limit(limit)
end
# This computes a toot id such that:
# - the toot would be old enough to be candidate for deletion
# - there are at most EARLY_SEARCH_CUTOFF toots between the last inspected toot and this one
#
# The idea is to limit expensive SQL queries when an account has lots of toots excluded from
# deletion, while not starting anew on each run.
def compute_cutoff_id
min_id = last_inspected || 0
max_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)
subquery = account.statuses.where(Status.arel_table[:id].gteq(min_id)).where(Status.arel_table[:id].lteq(max_id))
subquery = subquery.select(:id).reorder(id: :asc).limit(EARLY_SEARCH_CUTOFF)
# We're textually interpolating a subquery here as ActiveRecord seem to not provide
# a way to apply the limit to the subquery
Status.connection.execute("SELECT MAX(id) FROM (#{subquery.to_sql}) t").values.first.first
end
# The most important thing about `last_inspected` is that any toot older than it is guaranteed
# not to be kept by the policy regardless of its age.
def record_last_inspected(last_id)
redis.set("account_cleanup:#{account.id}", last_id, ex: 1.week.seconds)
end
def last_inspected
redis.get("account_cleanup:#{account.id}")&.to_i
end
def invalidate_last_inspected(status, action)
last_value = last_inspected
return if last_value.nil? || status.id > last_value || status.account_id != account_id
case action
when :unbookmark
return unless keep_self_bookmark?
when :unfav
return unless keep_self_fav?
when :unpin
return unless keep_pinned?
end
record_last_inspected(status.id)
end
private
def update_last_inspected
if EXCEPTION_BOOLS.map { |name| attribute_change_to_be_saved(name) }.compact.include?([true, false])
# Policy has been widened in such a way that any previously-inspected status
# may need to be deleted, so we'll have to start again.
redis.del("account_cleanup:#{account.id}")
end
if EXCEPTION_THRESHOLDS.map { |name| attribute_change_to_be_saved(name) }.compact.any? { |old, new| old.present? && (new.nil? || new > old) }
redis.del("account_cleanup:#{account.id}")
end
end
def validate_local_account
errors.add(:account, :invalid) unless account&.local?
end
def without_direct_scope
Status.where.not(visibility: :direct)
end
def old_enough_scope(max_id = nil)
# Filtering on `id` rather than `min_status_age` ago will treat
# non-snowflake statuses as older than they really are, but Mastodon
# has switched to snowflake IDs significantly over 2 years ago anyway.
max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min
Status.where(Status.arel_table[:id].lteq(max_id))
end
def without_self_fav_scope
Status.where('NOT EXISTS (SELECT * FROM favourites fav WHERE fav.account_id = statuses.account_id AND fav.status_id = statuses.id)')
end
def without_self_bookmark_scope
Status.where('NOT EXISTS (SELECT * FROM bookmarks bookmark WHERE bookmark.account_id = statuses.account_id AND bookmark.status_id = statuses.id)')
end
def without_pinned_scope
Status.where('NOT EXISTS (SELECT * FROM status_pins pin WHERE pin.account_id = statuses.account_id AND pin.status_id = statuses.id)')
end
def without_media_scope
Status.where('NOT EXISTS (SELECT * FROM media_attachments media WHERE media.status_id = statuses.id)')
end
def without_poll_scope
Status.where(poll_id: nil)
end
def without_popular_scope
scope = Status.left_joins(:status_stat)
scope = scope.where('COALESCE(status_stats.reblogs_count, 0) <= ?', min_reblogs) unless min_reblogs.nil?
scope = scope.where('COALESCE(status_stats.favourites_count, 0) <= ?', min_favs) unless min_favs.nil?
scope
end
end

@ -23,4 +23,12 @@ class Bookmark < ApplicationRecord
before_validation do
self.status = status.reblog if status&.reblog?
end
after_destroy :invalidate_cleanup_info
def invalidate_cleanup_info
return unless status&.account_id == account_id && account.local?
account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unbookmark)
end
end

@ -66,5 +66,8 @@ module AccountAssociations
# Follow recommendations
has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy
# Account statuses cleanup policy
has_one :statuses_cleanup_policy, class_name: 'AccountStatusesCleanupPolicy', inverse_of: :account, dependent: :destroy
end
end

@ -81,6 +81,9 @@ module AccountInteractions
has_many :following, -> { order('follows.id desc') }, through: :active_relationships, source: :target_account
has_many :followers, -> { order('follows.id desc') }, through: :passive_relationships, source: :account
# Account notes
has_many :account_notes, dependent: :destroy
# Block relationships
has_many :block_relationships, class_name: 'Block', foreign_key: 'account_id', dependent: :destroy
has_many :blocking, -> { order('blocks.id desc') }, through: :block_relationships, source: :target_account

@ -28,6 +28,7 @@ class Favourite < ApplicationRecord
after_create :increment_cache_counters
after_destroy :decrement_cache_counters
after_destroy :invalidate_cleanup_info
private
@ -39,4 +40,10 @@ class Favourite < ApplicationRecord
return if association(:status).loaded? && status.marked_for_destruction?
status&.decrement_count!(:favourites_count)
end
def invalidate_cleanup_info
return unless status&.account_id == account_id && account.local?
account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unfav)
end
end

@ -15,4 +15,12 @@ class StatusPin < ApplicationRecord
belongs_to :status
validates_with StatusPinValidator
after_destroy :invalidate_cleanup_info
def invalidate_cleanup_info
return unless status&.account_id == account_id && account.local?
account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unpin)
end
end

@ -0,0 +1,27 @@
# frozen_string_literal: true
class AccountStatusesCleanupService < BaseService
# @param [AccountStatusesCleanupPolicy] account_policy
# @param [Integer] budget
# @return [Integer]
def call(account_policy, budget = 50)
return 0 unless account_policy.enabled?
cutoff_id = account_policy.compute_cutoff_id
return 0 if cutoff_id.blank?
num_deleted = 0
last_deleted = nil
account_policy.statuses_to_delete(budget, cutoff_id, account_policy.last_inspected).reorder(nil).find_each(order: :asc) do |status|
status.discard
RemovalWorker.perform_async(status.id, redraft: false)
num_deleted += 1
last_deleted = status.id
end
account_policy.record_last_inspected(last_deleted.presence || cutoff_id)
num_deleted
end
end

@ -4,6 +4,7 @@ class DeleteAccountService < BaseService
include Payloadable
ASSOCIATIONS_ON_SUSPEND = %w(
account_notes
account_pins
active_relationships
aliases
@ -34,6 +35,7 @@ class DeleteAccountService < BaseService
# by foreign keys, making them safe to delete without loading
# into memory
ASSOCIATIONS_WITHOUT_SIDE_EFFECTS = %w(
account_notes
account_pins
aliases
conversation_mutes

@ -0,0 +1,45 @@
- content_for :page_title do
= t('settings.statuses_cleanup')
- content_for :heading_actions do
= button_tag t('generic.save_changes'), class: 'button', form: 'edit_policy'
= simple_form_for @policy, url: statuses_cleanup_path, method: :put, html: { id: 'edit_policy' } do |f|
.fields-row
.fields-row__column.fields-row__column-6.fields-group
= f.input :enabled, as: :boolean, wrapper: :with_label, label: t('statuses_cleanup.enabled'), hint: t('statuses_cleanup.enabled_hint')
.fields-row__column.fields-row__column-6.fields-group
= f.input :min_status_age, wrapper: :with_label, label: t('statuses_cleanup.min_age_label'), collection: AccountStatusesCleanupPolicy::ALLOWED_MIN_STATUS_AGE.map(&:to_i), label_method: lambda { |i| t("statuses_cleanup.min_age.#{i}") }, include_blank: false, hint: false
.flash-message= t('statuses_cleanup.explanation')
%h4= t('statuses_cleanup.exceptions')
.fields-row
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_pinned, wrapper: :with_label, label: t('statuses_cleanup.keep_pinned'), hint: t('statuses_cleanup.keep_pinned_hint')
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_direct, wrapper: :with_label, label: t('statuses_cleanup.keep_direct'), hint: t('statuses_cleanup.keep_direct_hint')
.fields-row
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_self_fav, wrapper: :with_label, label: t('statuses_cleanup.keep_self_fav'), hint: t('statuses_cleanup.keep_self_fav_hint')
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_self_bookmark, wrapper: :with_label, label: t('statuses_cleanup.keep_self_bookmark'), hint: t('statuses_cleanup.keep_self_bookmark_hint')
.fields-row
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_polls, wrapper: :with_label, label: t('statuses_cleanup.keep_polls'), hint: t('statuses_cleanup.keep_polls_hint')
.fields-row__column.fields-row__column-6.fields-group
= f.input :keep_media, wrapper: :with_label, label: t('statuses_cleanup.keep_media'), hint: t('statuses_cleanup.keep_media_hint')
%h4= t('statuses_cleanup.interaction_exceptions')
.fields-row
.fields-row__column.fields-row__column-6.fields-group
= f.input :min_favs, wrapper: :with_label, label: t('statuses_cleanup.min_favs'), hint: t('statuses_cleanup.min_favs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_favs') }
.fields-row__column.fields-row__column-6.fields-group
= f.input :min_reblogs, wrapper: :with_label, label: t('statuses_cleanup.min_reblogs'), hint: t('statuses_cleanup.min_reblogs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_reblogs') }
.flash-message= t('statuses_cleanup.interaction_exceptions_explanation')

@ -47,7 +47,7 @@ class MoveWorker
def copy_account_notes!
AccountNote.where(target_account: @source_account).find_each do |note|
text = I18n.with_locale(note.account.user.locale || I18n.default_locale) do
text = I18n.with_locale(note.account.user&.locale || I18n.default_locale) do
I18n.t('move_handler.copy_account_note_text', acct: @source_account.acct)
end
@ -84,7 +84,7 @@ class MoveWorker
def add_account_note_if_needed!(account, id)
unless AccountNote.where(account: account, target_account: @target_account).exists?
text = I18n.with_locale(account.user.locale || I18n.default_locale) do
text = I18n.with_locale(account.user&.locale || I18n.default_locale) do
I18n.t(id, acct: @source_account.acct)
end
AccountNote.create!(account: account, target_account: @target_account, comment: text)

@ -0,0 +1,96 @@
# frozen_string_literal: true
class Scheduler::AccountsStatusesCleanupScheduler
include Sidekiq::Worker
# This limit is mostly to be nice to the fediverse at large and not
# generate too much traffic.
# This also helps limiting the running time of the scheduler itself.
MAX_BUDGET = 50
# This is an attempt to spread the load across instances, as various
# accounts are likely to have various followers.
PER_ACCOUNT_BUDGET = 5
# This is an attempt to limit the workload generated by status removal
# jobs to something the particular instance can handle.
PER_THREAD_BUDGET = 5
# Those avoid loading an instance that is already under load
MAX_DEFAULT_SIZE = 2
MAX_DEFAULT_LATENCY = 5
MAX_PUSH_SIZE = 5
MAX_PUSH_LATENCY = 10
# 'pull' queue has lower priority jobs, and it's unlikely that pushing
# deletes would cause much issues with this queue if it didn't cause issues
# with default and push. Yet, do not enqueue deletes if the instance is
# lagging behind too much.
MAX_PULL_SIZE = 500
MAX_PULL_LATENCY = 300
# This is less of an issue in general, but deleting old statuses is likely
# to cause delivery errors, and thus increase the number of jobs to be retried.
# This doesn't directly translate to load, but connection errors and a high
# number of dead instances may lead to this spiraling out of control if
# unchecked.
MAX_RETRY_SIZE = 50_000
sidekiq_options retry: 0, lock: :until_executed
def perform
return if under_load?
budget = compute_budget
first_policy_id = last_processed_id
loop do
num_processed_accounts = 0
scope = AccountStatusesCleanupPolicy.where(enabled: true)
scope.where(Account.arel_table[:id].gt(first_policy_id)) if first_policy_id.present?
scope.find_each(order: :asc) do |policy|
num_deleted = AccountStatusesCleanupService.new.call(policy, [budget, PER_ACCOUNT_BUDGET].min)
num_processed_accounts += 1 unless num_deleted.zero?
budget -= num_deleted
if budget.zero?
save_last_processed_id(policy.id)
break
end
end
# The idea here is to loop through all policies at least once until the budget is exhausted
# and start back after the last processed account otherwise
break if budget.zero? || (num_processed_accounts.zero? && first_policy_id.nil?)
first_policy_id = nil
end
end
def compute_budget
threads = Sidekiq::ProcessSet.new.filter { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum
[PER_THREAD_BUDGET * threads, MAX_BUDGET].min
end
def under_load?
return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE
queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY)
end
private
def queue_under_load?(name, max_size, max_latency)
queue = Sidekiq::Queue.new(name)
queue.size > max_size || queue.latency > max_latency
end
def last_processed_id
Redis.current.get('account_statuses_cleanup_scheduler:last_account_id')
end
def save_last_processed_id(id)
if id.nil?
Redis.current.del('account_statuses_cleanup_scheduler:last_account_id')
else
Redis.current.set('account_statuses_cleanup_scheduler:last_account_id', id, ex: 1.hour.seconds)
end
end
end

@ -1254,6 +1254,7 @@ en:
preferences: Preferences
profile: Profile
relationships: Follows and followers
statuses_cleanup: Automated post deletion
two_factor_authentication: Two-factor Auth
webauthn_authentication: Security keys
statuses:
@ -1305,6 +1306,40 @@ en:
public_long: Everyone can see
unlisted: Unlisted
unlisted_long: Everyone can see, but not listed on public timelines
statuses_cleanup:
enabled: Automatically delete old posts
enabled_hint: Automatically deletes your posts once they reach a specified age threshold, unless they match one of the exceptions below
exceptions: Exceptions
explanation: Because deleting posts is an expensive operation, this is done slowly over time when the server is not otherwise busy. For this reason, your posts may be deleted a while after they reach the age threshold.
ignore_favs: Ignore favourites
ignore_reblogs: Ignore boosts
interaction_exceptions: Exceptions based on interactions
interaction_exceptions_explanation: Note that there is no guarantee for posts to be deleted if they go below the favourite or boost threshold after having once gone over them.
keep_direct: Keep direct messages
keep_direct_hint: Doesn't delete any of your direct messages
keep_media: Keep posts with media attachments
keep_media_hint: Doesn't delete any of your posts that have media attachments
keep_pinned: Keep pinned posts
keep_pinned_hint: Doesn't delete any of your pinned posts
keep_polls: Keep polls
keep_polls_hint: Doesn't delete any of your polls
keep_self_bookmark: Keep posts you bookmarked
keep_self_bookmark_hint: Doesn't delete your own posts if you have bookmarked them
keep_self_fav: Keep posts you favourited
keep_self_fav_hint: Doesn't delete your own posts if you have favourited them
min_age:
'1209600': 2 weeks
'15778476': 6 months
'2629746': 1 month
'31556952': 1 year
'5259492': 2 months
'63113904': 2 years
'7889238': 3 months
min_age_label: Age threshold
min_favs: Keep posts favourited more than
min_favs_hint: Doesn't delete any of your posts that has received more than this amount of favourites. Leave blank to delete posts regardless of their number of favourites
min_reblogs: Keep posts boosted more than
min_reblogs_hint: Doesn't delete any of your posts that has been boosted more than this number of times. Leave blank to delete posts regardless of their number of boosts
stream_entries:
pinned: Pinned post
reblogged: boosted

@ -24,6 +24,7 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_url, if: -> { current_user.functional? }
n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_url, if: -> { current_user.functional? }
n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s|
s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities}

@ -178,6 +178,7 @@ Rails.application.routes.draw do
resources :invites, only: [:index, :create, :destroy]
resources :filters, except: [:show]
resource :relationships, only: [:show, :update]
resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
get '/public', to: 'public_timelines#show', as: :public_timeline
get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy

@ -57,3 +57,7 @@
cron: '0 * * * *'
class: Scheduler::InstanceRefreshScheduler
queue: scheduler
accounts_statuses_cleanup_scheduler:
interval: 1 minute
class: Scheduler::AccountsStatusesCleanupScheduler
queue: scheduler

@ -0,0 +1,20 @@
class CreateAccountStatusesCleanupPolicies < ActiveRecord::Migration[6.1]
def change
create_table :account_statuses_cleanup_policies do |t|
t.belongs_to :account, null: false, foreign_key: { on_delete: :cascade }
t.boolean :enabled, null: false, default: true
t.integer :min_status_age, null: false, default: 2.weeks.seconds
t.boolean :keep_direct, null: false, default: true
t.boolean :keep_pinned, null: false, default: true
t.boolean :keep_polls, null: false, default: false
t.boolean :keep_media, null: false, default: false
t.boolean :keep_self_fav, null: false, default: true
t.boolean :keep_self_bookmark, null: false, default: true
t.integer :min_favs, null: true
t.integer :min_reblogs, null: true
t.timestamps
end
end
end

@ -0,0 +1,21 @@
# frozen_string_literal: true
class ClearOrphanedAccountNotes < ActiveRecord::Migration[5.2]
class Account < ApplicationRecord
# Dummy class, to make migration possible across version changes
end
class AccountNote < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account
belongs_to :target_account, class_name: 'Account'
end
def up
AccountNote.where('NOT EXISTS (SELECT * FROM users u WHERE u.account_id = account_notes.account_id)').in_batches.delete_all
end
def down
# nothing to do
end
end

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_06_30_000137) do
ActiveRecord::Schema.define(version: 2021_08_08_071221) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -114,6 +114,23 @@ ActiveRecord::Schema.define(version: 2021_06_30_000137) do
t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
end
create_table "account_statuses_cleanup_policies", force: :cascade do |t|
t.bigint "account_id", null: false
t.boolean "enabled", default: true, null: false
t.integer "min_status_age", default: 1209600, null: false
t.boolean "keep_direct", default: true, null: false
t.boolean "keep_pinned", default: true, null: false
t.boolean "keep_polls", default: false, null: false
t.boolean "keep_media", default: false, null: false
t.boolean "keep_self_fav", default: true, null: false
t.boolean "keep_self_bookmark", default: true, null: false
t.integer "min_favs"
t.integer "min_reblogs"
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.index ["account_id"], name: "index_account_statuses_cleanup_policies_on_account_id"
end
create_table "account_warning_presets", force: :cascade do |t|
t.text "text", default: "", null: false
t.datetime "created_at", null: false
@ -986,6 +1003,7 @@ ActiveRecord::Schema.define(version: 2021_06_30_000137) do
add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
add_foreign_key "account_pins", "accounts", on_delete: :cascade
add_foreign_key "account_stats", "accounts", on_delete: :cascade
add_foreign_key "account_statuses_cleanup_policies", "accounts", on_delete: :cascade
add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade
add_foreign_key "account_warnings", "accounts", on_delete: :nullify
add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify

@ -138,10 +138,11 @@ module Mastodon::Snowflake
end
end
def id_at(timestamp)
id = timestamp.to_i * 1000 + rand(1000)
def id_at(timestamp, with_random: true)
id = timestamp.to_i * 1000
id += rand(1000) if with_random
id = id << 16
id += rand(2**16)
id += rand(2**16) if with_random
id
end

@ -19,7 +19,7 @@ module Paperclip
metadata = VideoMetadataExtractor.new(@file.path)
unless metadata.valid?
log("Unsupported file #{@file.path}")
Paperclip.log("Unsupported file #{@file.path}")
return File.open(@file.path)
end

@ -60,13 +60,13 @@
},
"private": true,
"dependencies": {
"@babel/core": "^7.14.6",
"@babel/core": "^7.14.8",
"@babel/plugin-proposal-decorators": "^7.14.5",
"@babel/plugin-transform-react-inline-elements": "^7.14.5",
"@babel/plugin-transform-runtime": "^7.14.5",
"@babel/preset-env": "^7.14.7",
"@babel/preset-env": "^7.15.0",
"@babel/preset-react": "^7.14.5",
"@babel/runtime": "^7.14.6",
"@babel/runtime": "^7.14.8",
"@gamestdio/websocket": "^0.3.2",
"@github/webauthn-json": "^0.5.7",
"@rails/ujs": "^6.1.4",
@ -89,7 +89,7 @@
"css-loader": "^5.2.7",
"cssnano": "^4.1.11",
"detect-passive-events": "^2.0.3",
"dotenv": "^9.0.2",
"dotenv": "^10.0.0",
"emoji-mart": "^3.0.1",
"es6-symbol": "^3.1.3",
"escape-html": "^1.0.3",
@ -114,7 +114,7 @@
"marky": "^1.2.2",
"mini-css-extract-plugin": "^1.6.2",
"mkdirp": "^1.0.4",
"npmlog": "^4.1.2",
"npmlog": "^5.0.0",
"object-assign": "^4.1.1",
"object-fit-images": "^3.2.3",
"object.values": "^1.1.3",
@ -149,12 +149,12 @@
"redux": "^4.1.0",
"redux-immutable": "^4.0.0",
"redux-thunk": "^2.2.0",
"regenerator-runtime": "^0.13.7",
"regenerator-runtime": "^0.13.9",
"rellax": "^1.12.1",
"requestidlecallback": "^0.3.0",
"reselect": "^4.0.0",
"rimraf": "^3.0.2",
"sass": "^1.35.2",
"sass": "^1.37.0",
"sass-loader": "^10.2.0",
"stacktrace-js": "^2.0.2",
"stringz": "^2.1.0",
@ -171,14 +171,14 @@
"webpack-cli": "^3.3.12",
"webpack-merge": "^5.8.0",
"wicg-inert": "^3.1.1",
"ws": "^7.5.3"
"ws": "^8.0.0"
},
"devDependencies": {
"@testing-library/jest-dom": "^5.14.1",
"@testing-library/react": "^11.2.7",
"babel-eslint": "^10.1.0",
"babel-jest": "^27.0.6",
"eslint": "^7.31.0",
"eslint": "^7.32.0",
"eslint-plugin-import": "~2.23.4",
"eslint-plugin-jsx-a11y": "~6.4.1",
"eslint-plugin-promise": "~5.1.0",

@ -0,0 +1,27 @@
require 'rails_helper'
RSpec.describe StatusesCleanupController, type: :controller do
render_views
before do
@user = Fabricate(:user)
sign_in @user, scope: :user
end
describe "GET #show" do
it "returns http success" do
get :show
expect(response).to have_http_status(200)
end
end
describe 'PUT #update' do
it 'updates the account status cleanup policy' do
put :update, params: { account_statuses_cleanup_policy: { enabled: true, min_status_age: 2.weeks.seconds, keep_direct: false, keep_polls: true } }
expect(response).to redirect_to(statuses_cleanup_path)
expect(@user.account.statuses_cleanup_policy.enabled).to eq true
expect(@user.account.statuses_cleanup_policy.keep_direct).to eq false
expect(@user.account.statuses_cleanup_policy.keep_polls).to eq true
end
end
end

@ -0,0 +1,3 @@
Fabricator(:account_statuses_cleanup_policy) do
account
end

@ -0,0 +1,546 @@
require 'rails_helper'
RSpec.describe AccountStatusesCleanupPolicy, type: :model do
let(:account) { Fabricate(:account, username: 'alice', domain: nil) }
describe 'validation' do
it 'disallow remote accounts' do
account.update(domain: 'example.com')
account_statuses_cleanup_policy = Fabricate.build(:account_statuses_cleanup_policy, account: account)
account_statuses_cleanup_policy.valid?
expect(account_statuses_cleanup_policy).to model_have_error_on_field(:account)
end
end
describe 'save hooks' do
context 'when widening a policy' do
let!(:account_statuses_cleanup_policy) do
Fabricate(:account_statuses_cleanup_policy,
account: account,
keep_direct: true,
keep_pinned: true,
keep_polls: true,
keep_media: true,
keep_self_fav: true,
keep_self_bookmark: true,
min_favs: 1,
min_reblogs: 1
)
end
before do
account_statuses_cleanup_policy.record_last_inspected(42)
end
it 'invalidates last_inspected when widened because of keep_direct' do
account_statuses_cleanup_policy.keep_direct = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of keep_pinned' do
account_statuses_cleanup_policy.keep_pinned = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of keep_polls' do
account_statuses_cleanup_policy.keep_polls = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of keep_media' do
account_statuses_cleanup_policy.keep_media = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of keep_self_fav' do
account_statuses_cleanup_policy.keep_self_fav = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of keep_self_bookmark' do
account_statuses_cleanup_policy.keep_self_bookmark = false
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of higher min_favs' do
account_statuses_cleanup_policy.min_favs = 5
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of disabled min_favs' do
account_statuses_cleanup_policy.min_favs = nil
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of higher min_reblogs' do
account_statuses_cleanup_policy.min_reblogs = 5
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
it 'invalidates last_inspected when widened because of disable min_reblogs' do
account_statuses_cleanup_policy.min_reblogs = nil
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to be nil
end
end
context 'when narrowing a policy' do
let!(:account_statuses_cleanup_policy) do
Fabricate(:account_statuses_cleanup_policy,
account: account,
keep_direct: false,
keep_pinned: false,
keep_polls: false,
keep_media: false,
keep_self_fav: false,
keep_self_bookmark: false,
min_favs: nil,
min_reblogs: nil
)
end
it 'does not unnecessarily invalidate last_inspected' do
account_statuses_cleanup_policy.record_last_inspected(42)
account_statuses_cleanup_policy.keep_direct = true
account_statuses_cleanup_policy.keep_pinned = true
account_statuses_cleanup_policy.keep_polls = true
account_statuses_cleanup_policy.keep_media = true
account_statuses_cleanup_policy.keep_self_fav = true
account_statuses_cleanup_policy.keep_self_bookmark = true
account_statuses_cleanup_policy.min_favs = 5
account_statuses_cleanup_policy.min_reblogs = 5
account_statuses_cleanup_policy.save
expect(account_statuses_cleanup_policy.last_inspected).to eq 42
end
end
end
describe '#record_last_inspected' do
let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
it 'records the given id' do
account_statuses_cleanup_policy.record_last_inspected(42)
expect(account_statuses_cleanup_policy.last_inspected).to eq 42
end
end
describe '#invalidate_last_inspected' do
let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
let(:status) { Fabricate(:status, id: 10, account: account) }
subject { account_statuses_cleanup_policy.invalidate_last_inspected(status, action) }
before do
account_statuses_cleanup_policy.record_last_inspected(42)
end
context 'when the action is :unbookmark' do
let(:action) { :unbookmark }
context 'when the policy is not to keep self-bookmarked toots' do
before do
account_statuses_cleanup_policy.keep_self_bookmark = false
end
it 'does not change the recorded id' do