diff --git a/app/controllers/statuses_cleanup_controller.rb b/app/controllers/statuses_cleanup_controller.rb new file mode 100644 index 000000000..be234cdcb --- /dev/null +++ b/app/controllers/statuses_cleanup_controller.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class StatusesCleanupController < ApplicationController + layout 'admin' + + before_action :authenticate_user! + before_action :set_policy + before_action :set_body_classes + + def show; end + + def update + if @policy.update(resource_params) + redirect_to statuses_cleanup_path, notice: I18n.t('generic.changes_saved_msg') + else + render action: :show + end + rescue ActionController::ParameterMissing + # Do nothing + end + + private + + def set_policy + @policy = current_account.statuses_cleanup_policy || current_account.build_statuses_cleanup_policy(enabled: false) + end + + def resource_params + params.require(:account_statuses_cleanup_policy).permit(:enabled, :min_status_age, :keep_direct, :keep_pinned, :keep_polls, :keep_media, :keep_self_fav, :keep_self_bookmark, :min_favs, :min_reblogs) + end + + def set_body_classes + @body_classes = 'admin' + end +end diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb new file mode 100644 index 000000000..705ccff54 --- /dev/null +++ b/app/models/account_statuses_cleanup_policy.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: account_statuses_cleanup_policies +# +# id :bigint not null, primary key +# account_id :bigint not null +# enabled :boolean default(TRUE), not null +# min_status_age :integer default(1209600), not null +# keep_direct :boolean default(TRUE), not null +# keep_pinned :boolean default(TRUE), not null +# keep_polls :boolean default(FALSE), not null +# keep_media :boolean default(FALSE), not null +# keep_self_fav :boolean default(TRUE), not null +# keep_self_bookmark :boolean default(TRUE), not null +# min_favs :integer +# min_reblogs :integer +# created_at :datetime not null +# updated_at :datetime not null +# +class AccountStatusesCleanupPolicy < ApplicationRecord + include Redisable + + ALLOWED_MIN_STATUS_AGE = [ + 2.weeks.seconds, + 1.month.seconds, + 2.months.seconds, + 3.months.seconds, + 6.months.seconds, + 1.year.seconds, + 2.years.seconds, + ].freeze + + EXCEPTION_BOOLS = %w(keep_direct keep_pinned keep_polls keep_media keep_self_fav keep_self_bookmark).freeze + EXCEPTION_THRESHOLDS = %w(min_favs min_reblogs).freeze + + # Depending on the cleanup policy, the query to discover the next + # statuses to delete my get expensive if the account has a lot of old + # statuses otherwise excluded from deletion by the other exceptions. + # + # Therefore, `EARLY_SEARCH_CUTOFF` is meant to be the maximum number of + # old statuses to be considered for deletion prior to checking exceptions. + # + # This is used in `compute_cutoff_id` to provide a `max_id` to + # `statuses_to_delete`. + EARLY_SEARCH_CUTOFF = 5_000 + + belongs_to :account + + validates :min_status_age, inclusion: { in: ALLOWED_MIN_STATUS_AGE } + validates :min_favs, numericality: { greater_than_or_equal_to: 1, allow_nil: true } + validates :min_reblogs, numericality: { greater_than_or_equal_to: 1, allow_nil: true } + validate :validate_local_account + + before_save :update_last_inspected + + def statuses_to_delete(limit = 50, max_id = nil, min_id = nil) + scope = account.statuses + scope.merge!(old_enough_scope(max_id)) + scope = scope.where(Status.arel_table[:id].gteq(min_id)) if min_id.present? + scope.merge!(without_popular_scope) unless min_favs.nil? && min_reblogs.nil? + scope.merge!(without_direct_scope) if keep_direct? + scope.merge!(without_pinned_scope) if keep_pinned? + scope.merge!(without_poll_scope) if keep_polls? + scope.merge!(without_media_scope) if keep_media? + scope.merge!(without_self_fav_scope) if keep_self_fav? + scope.merge!(without_self_bookmark_scope) if keep_self_bookmark? + + scope.reorder(id: :asc).limit(limit) + end + + # This computes a toot id such that: + # - the toot would be old enough to be candidate for deletion + # - there are at most EARLY_SEARCH_CUTOFF toots between the last inspected toot and this one + # + # The idea is to limit expensive SQL queries when an account has lots of toots excluded from + # deletion, while not starting anew on each run. + def compute_cutoff_id + min_id = last_inspected || 0 + max_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false) + subquery = account.statuses.where(Status.arel_table[:id].gteq(min_id)).where(Status.arel_table[:id].lteq(max_id)) + subquery = subquery.select(:id).reorder(id: :asc).limit(EARLY_SEARCH_CUTOFF) + + # We're textually interpolating a subquery here as ActiveRecord seem to not provide + # a way to apply the limit to the subquery + Status.connection.execute("SELECT MAX(id) FROM (#{subquery.to_sql}) t").values.first.first + end + + # The most important thing about `last_inspected` is that any toot older than it is guaranteed + # not to be kept by the policy regardless of its age. + def record_last_inspected(last_id) + redis.set("account_cleanup:#{account.id}", last_id, ex: 1.week.seconds) + end + + def last_inspected + redis.get("account_cleanup:#{account.id}")&.to_i + end + + def invalidate_last_inspected(status, action) + last_value = last_inspected + return if last_value.nil? || status.id > last_value || status.account_id != account_id + + case action + when :unbookmark + return unless keep_self_bookmark? + when :unfav + return unless keep_self_fav? + when :unpin + return unless keep_pinned? + end + + record_last_inspected(status.id) + end + + private + + def update_last_inspected + if EXCEPTION_BOOLS.map { |name| attribute_change_to_be_saved(name) }.compact.include?([true, false]) + # Policy has been widened in such a way that any previously-inspected status + # may need to be deleted, so we'll have to start again. + redis.del("account_cleanup:#{account.id}") + end + if EXCEPTION_THRESHOLDS.map { |name| attribute_change_to_be_saved(name) }.compact.any? { |old, new| old.present? && (new.nil? || new > old) } + redis.del("account_cleanup:#{account.id}") + end + end + + def validate_local_account + errors.add(:account, :invalid) unless account&.local? + end + + def without_direct_scope + Status.where.not(visibility: :direct) + end + + def old_enough_scope(max_id = nil) + # Filtering on `id` rather than `min_status_age` ago will treat + # non-snowflake statuses as older than they really are, but Mastodon + # has switched to snowflake IDs significantly over 2 years ago anyway. + max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min + Status.where(Status.arel_table[:id].lteq(max_id)) + end + + def without_self_fav_scope + Status.where('NOT EXISTS (SELECT * FROM favourites fav WHERE fav.account_id = statuses.account_id AND fav.status_id = statuses.id)') + end + + def without_self_bookmark_scope + Status.where('NOT EXISTS (SELECT * FROM bookmarks bookmark WHERE bookmark.account_id = statuses.account_id AND bookmark.status_id = statuses.id)') + end + + def without_pinned_scope + Status.where('NOT EXISTS (SELECT * FROM status_pins pin WHERE pin.account_id = statuses.account_id AND pin.status_id = statuses.id)') + end + + def without_media_scope + Status.where('NOT EXISTS (SELECT * FROM media_attachments media WHERE media.status_id = statuses.id)') + end + + def without_poll_scope + Status.where(poll_id: nil) + end + + def without_popular_scope + scope = Status.left_joins(:status_stat) + scope = scope.where('COALESCE(status_stats.reblogs_count, 0) <= ?', min_reblogs) unless min_reblogs.nil? + scope = scope.where('COALESCE(status_stats.favourites_count, 0) <= ?', min_favs) unless min_favs.nil? + scope + end +end diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb index 916261a17..f21ea714c 100644 --- a/app/models/bookmark.rb +++ b/app/models/bookmark.rb @@ -23,4 +23,12 @@ class Bookmark < ApplicationRecord before_validation do self.status = status.reblog if status&.reblog? end + + after_destroy :invalidate_cleanup_info + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unbookmark) + end end diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index aaf371ebd..f2a4eae77 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -66,5 +66,8 @@ module AccountAssociations # Follow recommendations has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy + + # Account statuses cleanup policy + has_one :statuses_cleanup_policy, class_name: 'AccountStatusesCleanupPolicy', inverse_of: :account, dependent: :destroy end end diff --git a/app/models/favourite.rb b/app/models/favourite.rb index 35028b7dd..ca8bce146 100644 --- a/app/models/favourite.rb +++ b/app/models/favourite.rb @@ -28,6 +28,7 @@ class Favourite < ApplicationRecord after_create :increment_cache_counters after_destroy :decrement_cache_counters + after_destroy :invalidate_cleanup_info private @@ -39,4 +40,10 @@ class Favourite < ApplicationRecord return if association(:status).loaded? && status.marked_for_destruction? status&.decrement_count!(:favourites_count) end + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unfav) + end end diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb index afc76bded..93a0ea1c0 100644 --- a/app/models/status_pin.rb +++ b/app/models/status_pin.rb @@ -15,4 +15,12 @@ class StatusPin < ApplicationRecord belongs_to :status validates_with StatusPinValidator + + after_destroy :invalidate_cleanup_info + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unpin) + end end diff --git a/app/services/account_statuses_cleanup_service.rb b/app/services/account_statuses_cleanup_service.rb new file mode 100644 index 000000000..cbadecc63 --- /dev/null +++ b/app/services/account_statuses_cleanup_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AccountStatusesCleanupService < BaseService + # @param [AccountStatusesCleanupPolicy] account_policy + # @param [Integer] budget + # @return [Integer] + def call(account_policy, budget = 50) + return 0 unless account_policy.enabled? + + cutoff_id = account_policy.compute_cutoff_id + return 0 if cutoff_id.blank? + + num_deleted = 0 + last_deleted = nil + + account_policy.statuses_to_delete(budget, cutoff_id, account_policy.last_inspected).reorder(nil).find_each(order: :asc) do |status| + status.discard + RemovalWorker.perform_async(status.id, redraft: false) + num_deleted += 1 + last_deleted = status.id + end + + account_policy.record_last_inspected(last_deleted.presence || cutoff_id) + + num_deleted + end +end diff --git a/app/views/statuses_cleanup/show.html.haml b/app/views/statuses_cleanup/show.html.haml new file mode 100644 index 000000000..59de4b5aa --- /dev/null +++ b/app/views/statuses_cleanup/show.html.haml @@ -0,0 +1,45 @@ +- content_for :page_title do + = t('settings.statuses_cleanup') + +- content_for :heading_actions do + = button_tag t('generic.save_changes'), class: 'button', form: 'edit_policy' + += simple_form_for @policy, url: statuses_cleanup_path, method: :put, html: { id: 'edit_policy' } do |f| + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :enabled, as: :boolean, wrapper: :with_label, label: t('statuses_cleanup.enabled'), hint: t('statuses_cleanup.enabled_hint') + .fields-row__column.fields-row__column-6.fields-group + = f.input :min_status_age, wrapper: :with_label, label: t('statuses_cleanup.min_age_label'), collection: AccountStatusesCleanupPolicy::ALLOWED_MIN_STATUS_AGE.map(&:to_i), label_method: lambda { |i| t("statuses_cleanup.min_age.#{i}") }, include_blank: false, hint: false + + .flash-message= t('statuses_cleanup.explanation') + + %h4= t('statuses_cleanup.exceptions') + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_pinned, wrapper: :with_label, label: t('statuses_cleanup.keep_pinned'), hint: t('statuses_cleanup.keep_pinned_hint') + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_direct, wrapper: :with_label, label: t('statuses_cleanup.keep_direct'), hint: t('statuses_cleanup.keep_direct_hint') + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_self_fav, wrapper: :with_label, label: t('statuses_cleanup.keep_self_fav'), hint: t('statuses_cleanup.keep_self_fav_hint') + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_self_bookmark, wrapper: :with_label, label: t('statuses_cleanup.keep_self_bookmark'), hint: t('statuses_cleanup.keep_self_bookmark_hint') + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_polls, wrapper: :with_label, label: t('statuses_cleanup.keep_polls'), hint: t('statuses_cleanup.keep_polls_hint') + .fields-row__column.fields-row__column-6.fields-group + = f.input :keep_media, wrapper: :with_label, label: t('statuses_cleanup.keep_media'), hint: t('statuses_cleanup.keep_media_hint') + + %h4= t('statuses_cleanup.interaction_exceptions') + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :min_favs, wrapper: :with_label, label: t('statuses_cleanup.min_favs'), hint: t('statuses_cleanup.min_favs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_favs') } + .fields-row__column.fields-row__column-6.fields-group + = f.input :min_reblogs, wrapper: :with_label, label: t('statuses_cleanup.min_reblogs'), hint: t('statuses_cleanup.min_reblogs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_reblogs') } + + .flash-message= t('statuses_cleanup.interaction_exceptions_explanation') diff --git a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb new file mode 100644 index 000000000..f42d4bca6 --- /dev/null +++ b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +class Scheduler::AccountsStatusesCleanupScheduler + include Sidekiq::Worker + + # This limit is mostly to be nice to the fediverse at large and not + # generate too much traffic. + # This also helps limiting the running time of the scheduler itself. + MAX_BUDGET = 50 + + # This is an attempt to spread the load across instances, as various + # accounts are likely to have various followers. + PER_ACCOUNT_BUDGET = 5 + + # This is an attempt to limit the workload generated by status removal + # jobs to something the particular instance can handle. + PER_THREAD_BUDGET = 5 + + # Those avoid loading an instance that is already under load + MAX_DEFAULT_SIZE = 2 + MAX_DEFAULT_LATENCY = 5 + MAX_PUSH_SIZE = 5 + MAX_PUSH_LATENCY = 10 + # 'pull' queue has lower priority jobs, and it's unlikely that pushing + # deletes would cause much issues with this queue if it didn't cause issues + # with default and push. Yet, do not enqueue deletes if the instance is + # lagging behind too much. + MAX_PULL_SIZE = 500 + MAX_PULL_LATENCY = 300 + + # This is less of an issue in general, but deleting old statuses is likely + # to cause delivery errors, and thus increase the number of jobs to be retried. + # This doesn't directly translate to load, but connection errors and a high + # number of dead instances may lead to this spiraling out of control if + # unchecked. + MAX_RETRY_SIZE = 50_000 + + sidekiq_options retry: 0, lock: :until_executed + + def perform + return if under_load? + + budget = compute_budget + first_policy_id = last_processed_id + + loop do + num_processed_accounts = 0 + + scope = AccountStatusesCleanupPolicy.where(enabled: true) + scope.where(Account.arel_table[:id].gt(first_policy_id)) if first_policy_id.present? + scope.find_each(order: :asc) do |policy| + num_deleted = AccountStatusesCleanupService.new.call(policy, [budget, PER_ACCOUNT_BUDGET].min) + num_processed_accounts += 1 unless num_deleted.zero? + budget -= num_deleted + if budget.zero? + save_last_processed_id(policy.id) + break + end + end + + # The idea here is to loop through all policies at least once until the budget is exhausted + # and start back after the last processed account otherwise + break if budget.zero? || (num_processed_accounts.zero? && first_policy_id.nil?) + first_policy_id = nil + end + end + + def compute_budget + threads = Sidekiq::ProcessSet.new.filter { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum + [PER_THREAD_BUDGET * threads, MAX_BUDGET].min + end + + def under_load? + return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE + queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY) + end + + private + + def queue_under_load?(name, max_size, max_latency) + queue = Sidekiq::Queue.new(name) + queue.size > max_size || queue.latency > max_latency + end + + def last_processed_id + Redis.current.get('account_statuses_cleanup_scheduler:last_account_id') + end + + def save_last_processed_id(id) + if id.nil? + Redis.current.del('account_statuses_cleanup_scheduler:last_account_id') + else + Redis.current.set('account_statuses_cleanup_scheduler:last_account_id', id, ex: 1.hour.seconds) + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index af7266d86..be6052948 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1254,6 +1254,7 @@ en: preferences: Preferences profile: Profile relationships: Follows and followers + statuses_cleanup: Automated post deletion two_factor_authentication: Two-factor Auth webauthn_authentication: Security keys statuses: @@ -1305,6 +1306,40 @@ en: public_long: Everyone can see unlisted: Unlisted unlisted_long: Everyone can see, but not listed on public timelines + statuses_cleanup: + enabled: Automatically delete old posts + enabled_hint: Automatically deletes your posts once they reach a specified age threshold, unless they match one of the exceptions below + exceptions: Exceptions + explanation: Because deleting posts is an expensive operation, this is done slowly over time when the server is not otherwise busy. For this reason, your posts may be deleted a while after they reach the age threshold. + ignore_favs: Ignore favourites + ignore_reblogs: Ignore boosts + interaction_exceptions: Exceptions based on interactions + interaction_exceptions_explanation: Note that there is no guarantee for posts to be deleted if they go below the favourite or boost threshold after having once gone over them. + keep_direct: Keep direct messages + keep_direct_hint: Doesn't delete any of your direct messages + keep_media: Keep posts with media attachments + keep_media_hint: Doesn't delete any of your posts that have media attachments + keep_pinned: Keep pinned posts + keep_pinned_hint: Doesn't delete any of your pinned posts + keep_polls: Keep polls + keep_polls_hint: Doesn't delete any of your polls + keep_self_bookmark: Keep posts you bookmarked + keep_self_bookmark_hint: Doesn't delete your own posts if you have bookmarked them + keep_self_fav: Keep posts you favourited + keep_self_fav_hint: Doesn't delete your own posts if you have favourited them + min_age: + '1209600': 2 weeks + '15778476': 6 months + '2629746': 1 month + '31556952': 1 year + '5259492': 2 months + '63113904': 2 years + '7889238': 3 months + min_age_label: Age threshold + min_favs: Keep posts favourited more than + min_favs_hint: Doesn't delete any of your posts that has received more than this amount of favourites. Leave blank to delete posts regardless of their number of favourites + min_reblogs: Keep posts boosted more than + min_reblogs_hint: Doesn't delete any of your posts that has been boosted more than this number of times. Leave blank to delete posts regardless of their number of boosts stream_entries: pinned: Pinned post reblogged: boosted diff --git a/config/navigation.rb b/config/navigation.rb index 5d1f55d74..37bfd7549 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -18,6 +18,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_url, if: -> { current_user.functional? } n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? } + n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_url, if: -> { current_user.functional? } n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s| s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities} diff --git a/config/routes.rb b/config/routes.rb index 0c4b29546..8abc438fe 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -176,6 +176,7 @@ Rails.application.routes.draw do resources :invites, only: [:index, :create, :destroy] resources :filters, except: [:show] resource :relationships, only: [:show, :update] + resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update] get '/public', to: 'public_timelines#show', as: :public_timeline get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy diff --git a/config/sidekiq.yml b/config/sidekiq.yml index a8e4c7feb..eab74338e 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -57,3 +57,7 @@ cron: '0 * * * *' class: Scheduler::InstanceRefreshScheduler queue: scheduler + accounts_statuses_cleanup_scheduler: + interval: 1 minute + class: Scheduler::AccountsStatusesCleanupScheduler + queue: scheduler diff --git a/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb b/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb new file mode 100644 index 000000000..28cfb6ef5 --- /dev/null +++ b/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb @@ -0,0 +1,20 @@ +class CreateAccountStatusesCleanupPolicies < ActiveRecord::Migration[6.1] + def change + create_table :account_statuses_cleanup_policies do |t| + t.belongs_to :account, null: false, foreign_key: { on_delete: :cascade } + t.boolean :enabled, null: false, default: true + t.integer :min_status_age, null: false, default: 2.weeks.seconds + t.boolean :keep_direct, null: false, default: true + t.boolean :keep_pinned, null: false, default: true + t.boolean :keep_polls, null: false, default: false + t.boolean :keep_media, null: false, default: false + t.boolean :keep_self_fav, null: false, default: true + t.boolean :keep_self_bookmark, null: false, default: true + t.integer :min_favs, null: true + t.integer :min_reblogs, null: true + + t.timestamps + end + end +end + diff --git a/db/schema.rb b/db/schema.rb index a0a98eb03..2376afff7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -114,6 +114,23 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true end + create_table "account_statuses_cleanup_policies", force: :cascade do |t| + t.bigint "account_id", null: false + t.boolean "enabled", default: true, null: false + t.integer "min_status_age", default: 1209600, null: false + t.boolean "keep_direct", default: true, null: false + t.boolean "keep_pinned", default: true, null: false + t.boolean "keep_polls", default: false, null: false + t.boolean "keep_media", default: false, null: false + t.boolean "keep_self_fav", default: true, null: false + t.boolean "keep_self_bookmark", default: true, null: false + t.integer "min_favs" + t.integer "min_reblogs" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["account_id"], name: "index_account_statuses_cleanup_policies_on_account_id" + end + create_table "account_warning_presets", force: :cascade do |t| t.text "text", default: "", null: false t.datetime "created_at", null: false @@ -984,6 +1001,7 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade add_foreign_key "account_pins", "accounts", on_delete: :cascade add_foreign_key "account_stats", "accounts", on_delete: :cascade + add_foreign_key "account_statuses_cleanup_policies", "accounts", on_delete: :cascade add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade add_foreign_key "account_warnings", "accounts", on_delete: :nullify add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb index 9e5bc7383..8e2d82a97 100644 --- a/lib/mastodon/snowflake.rb +++ b/lib/mastodon/snowflake.rb @@ -138,10 +138,11 @@ module Mastodon::Snowflake end end - def id_at(timestamp) - id = timestamp.to_i * 1000 + rand(1000) + def id_at(timestamp, with_random: true) + id = timestamp.to_i * 1000 + id += rand(1000) if with_random id = id << 16 - id += rand(2**16) + id += rand(2**16) if with_random id end diff --git a/spec/controllers/statuses_cleanup_controller_spec.rb b/spec/controllers/statuses_cleanup_controller_spec.rb new file mode 100644 index 000000000..924709260 --- /dev/null +++ b/spec/controllers/statuses_cleanup_controller_spec.rb @@ -0,0 +1,27 @@ +require 'rails_helper' + +RSpec.describe StatusesCleanupController, type: :controller do + render_views + + before do + @user = Fabricate(:user) + sign_in @user, scope: :user + end + + describe "GET #show" do + it "returns http success" do + get :show + expect(response).to have_http_status(200) + end + end + + describe 'PUT #update' do + it 'updates the account status cleanup policy' do + put :update, params: { account_statuses_cleanup_policy: { enabled: true, min_status_age: 2.weeks.seconds, keep_direct: false, keep_polls: true } } + expect(response).to redirect_to(statuses_cleanup_path) + expect(@user.account.statuses_cleanup_policy.enabled).to eq true + expect(@user.account.statuses_cleanup_policy.keep_direct).to eq false + expect(@user.account.statuses_cleanup_policy.keep_polls).to eq true + end + end +end diff --git a/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb b/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb new file mode 100644 index 000000000..29cf1d133 --- /dev/null +++ b/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:account_statuses_cleanup_policy) do + account +end diff --git a/spec/models/account_statuses_cleanup_policy_spec.rb b/spec/models/account_statuses_cleanup_policy_spec.rb new file mode 100644 index 000000000..63e9c5d20 --- /dev/null +++ b/spec/models/account_statuses_cleanup_policy_spec.rb @@ -0,0 +1,546 @@ +require 'rails_helper' + +RSpec.describe AccountStatusesCleanupPolicy, type: :model do + let(:account) { Fabricate(:account, username: 'alice', domain: nil) } + + describe 'validation' do + it 'disallow remote accounts' do + account.update(domain: 'example.com') + account_statuses_cleanup_policy = Fabricate.build(:account_statuses_cleanup_policy, account: account) + account_statuses_cleanup_policy.valid? + expect(account_statuses_cleanup_policy).to model_have_error_on_field(:account) + end + end + + describe 'save hooks' do + context 'when widening a policy' do + let!(:account_statuses_cleanup_policy) do + Fabricate(:account_statuses_cleanup_policy, + account: account, + keep_direct: true, + keep_pinned: true, + keep_polls: true, + keep_media: true, + keep_self_fav: true, + keep_self_bookmark: true, + min_favs: 1, + min_reblogs: 1 + ) + end + + before do + account_statuses_cleanup_policy.record_last_inspected(42) + end + + it 'invalidates last_inspected when widened because of keep_direct' do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of keep_pinned' do + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of keep_polls' do + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of keep_media' do + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of keep_self_fav' do + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of keep_self_bookmark' do + account_statuses_cleanup_policy.keep_self_bookmark = false + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of higher min_favs' do + account_statuses_cleanup_policy.min_favs = 5 + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of disabled min_favs' do + account_statuses_cleanup_policy.min_favs = nil + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of higher min_reblogs' do + account_statuses_cleanup_policy.min_reblogs = 5 + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + + it 'invalidates last_inspected when widened because of disable min_reblogs' do + account_statuses_cleanup_policy.min_reblogs = nil + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to be nil + end + end + + context 'when narrowing a policy' do + let!(:account_statuses_cleanup_policy) do + Fabricate(:account_statuses_cleanup_policy, + account: account, + keep_direct: false, + keep_pinned: false, + keep_polls: false, + keep_media: false, + keep_self_fav: false, + keep_self_bookmark: false, + min_favs: nil, + min_reblogs: nil + ) + end + + it 'does not unnecessarily invalidate last_inspected' do + account_statuses_cleanup_policy.record_last_inspected(42) + account_statuses_cleanup_policy.keep_direct = true + account_statuses_cleanup_policy.keep_pinned = true + account_statuses_cleanup_policy.keep_polls = true + account_statuses_cleanup_policy.keep_media = true + account_statuses_cleanup_policy.keep_self_fav = true + account_statuses_cleanup_policy.keep_self_bookmark = true + account_statuses_cleanup_policy.min_favs = 5 + account_statuses_cleanup_policy.min_reblogs = 5 + account_statuses_cleanup_policy.save + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + end + + describe '#record_last_inspected' do + let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) } + + it 'records the given id' do + account_statuses_cleanup_policy.record_last_inspected(42) + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + + describe '#invalidate_last_inspected' do + let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) } + let(:status) { Fabricate(:status, id: 10, account: account) } + subject { account_statuses_cleanup_policy.invalidate_last_inspected(status, action) } + + before do + account_statuses_cleanup_policy.record_last_inspected(42) + end + + context 'when the action is :unbookmark' do + let(:action) { :unbookmark } + + context 'when the policy is not to keep self-bookmarked toots' do + before do + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not change the recorded id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + + context 'when the policy is to keep self-bookmarked toots' do + before do + account_statuses_cleanup_policy.keep_self_bookmark = true + end + + it 'records the older id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 10 + end + end + end + + context 'when the action is :unfav' do + let(:action) { :unfav } + + context 'when the policy is not to keep self-favourited toots' do + before do + account_statuses_cleanup_policy.keep_self_fav = false + end + + it 'does not change the recorded id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + + context 'when the policy is to keep self-favourited toots' do + before do + account_statuses_cleanup_policy.keep_self_fav = true + end + + it 'records the older id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 10 + end + end + end + + context 'when the action is :unpin' do + let(:action) { :unpin } + + context 'when the policy is not to keep pinned toots' do + before do + account_statuses_cleanup_policy.keep_pinned = false + end + + it 'does not change the recorded id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + + context 'when the policy is to keep pinned toots' do + before do + account_statuses_cleanup_policy.keep_pinned = true + end + + it 'records the older id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 10 + end + end + end + + context 'when the status is more recent than the recorded inspected id' do + let(:action) { :unfav } + let(:status) { Fabricate(:status, account: account) } + + it 'does not change the recorded id' do + subject + expect(account_statuses_cleanup_policy.last_inspected).to eq 42 + end + end + end + + describe '#compute_cutoff_id' do + let!(:unrelated_status) { Fabricate(:status, created_at: 3.years.ago) } + let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) } + + subject { account_statuses_cleanup_policy.compute_cutoff_id } + + context 'when the account has posted multiple toots' do + let!(:very_old_status) { Fabricate(:status, created_at: 3.years.ago, account: account) } + let!(:old_status) { Fabricate(:status, created_at: 3.weeks.ago, account: account) } + let!(:recent_status) { Fabricate(:status, created_at: 2.days.ago, account: account) } + + it 'returns the most recent id that is still below policy age' do + expect(subject).to eq old_status.id + end + end + + context 'when the account has not posted anything' do + it 'returns nil' do + expect(subject).to be_nil + end + end + end + + describe '#statuses_to_delete' do + let!(:unrelated_status) { Fabricate(:status, created_at: 3.years.ago) } + let!(:very_old_status) { Fabricate(:status, created_at: 3.years.ago, account: account) } + let!(:pinned_status) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:direct_message) { Fabricate(:status, created_at: 1.year.ago, account: account, visibility: :direct) } + let!(:self_faved) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:self_bookmarked) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:status_with_poll) { Fabricate(:status, created_at: 1.year.ago, account: account, poll_attributes: { account: account, voters_count: 0, options: ['a', 'b'], expires_in: 2.days }) } + let!(:status_with_media) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:faved4) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:faved5) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:reblogged4) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:reblogged5) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:recent_status) { Fabricate(:status, created_at: 2.days.ago, account: account) } + + let!(:media_attachment) { Fabricate(:media_attachment, account: account, status: status_with_media) } + let!(:status_pin) { Fabricate(:status_pin, account: account, status: pinned_status) } + let!(:favourite) { Fabricate(:favourite, account: account, status: self_faved) } + let!(:bookmark) { Fabricate(:bookmark, account: account, status: self_bookmarked) } + + let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) } + + subject { account_statuses_cleanup_policy.statuses_to_delete } + + before do + 4.times { faved4.increment_count!(:favourites_count) } + 5.times { faved5.increment_count!(:favourites_count) } + 4.times { reblogged4.increment_count!(:reblogs_count) } + 5.times { reblogged5.increment_count!(:reblogs_count) } + end + + context 'when passed a max_id' do + let!(:old_status) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:slightly_less_old_status) { Fabricate(:status, created_at: 6.months.ago, account: account) } + + subject { account_statuses_cleanup_policy.statuses_to_delete(50, old_status.id).pluck(:id) } + + it 'returns statuses including max_id' do + expect(subject).to include(old_status.id) + end + + it 'returns statuses including older than max_id' do + expect(subject).to include(very_old_status.id) + end + + it 'does not return statuses newer than max_id' do + expect(subject).to_not include(slightly_less_old_status.id) + end + end + + context 'when passed a min_id' do + let!(:old_status) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:slightly_less_old_status) { Fabricate(:status, created_at: 6.months.ago, account: account) } + + subject { account_statuses_cleanup_policy.statuses_to_delete(50, recent_status.id, old_status.id).pluck(:id) } + + it 'returns statuses including min_id' do + expect(subject).to include(old_status.id) + end + + it 'returns statuses including newer than max_id' do + expect(subject).to include(slightly_less_old_status.id) + end + + it 'does not return statuses older than min_id' do + expect(subject).to_not include(very_old_status.id) + end + end + + context 'when passed a low limit' do + it 'only returns the limited number of items' do + expect(account_statuses_cleanup_policy.statuses_to_delete(1).count).to eq 1 + end + end + + context 'when policy is set to keep statuses more recent than 2 years' do + before do + account_statuses_cleanup_policy.min_status_age = 2.years.seconds + end + + it 'does not return unrelated old status' do + expect(subject.pluck(:id)).to_not include(unrelated_status.id) + end + + it 'returns only oldest status for deletion' do + expect(subject.pluck(:id)).to eq [very_old_status.id] + end + end + + context 'when policy is set to keep DMs and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = true + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the old direct message for deletion' do + expect(subject.pluck(:id)).to_not include(direct_message.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep self-bookmarked toots and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = true + end + + it 'does not return the old self-bookmarked message for deletion' do + expect(subject.pluck(:id)).to_not include(self_bookmarked.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep self-faved toots and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = true + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the old self-bookmarked message for deletion' do + expect(subject.pluck(:id)).to_not include(self_faved.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep toots with media and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = true + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the old message with media for deletion' do + expect(subject.pluck(:id)).to_not include(status_with_media.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep toots with polls and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = true + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the old poll message for deletion' do + expect(subject.pluck(:id)).to_not include(status_with_poll.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep pinned toots and reject everything else' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = true + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the old pinned message for deletion' do + expect(subject.pluck(:id)).to_not include(pinned_status.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is to not keep any special messages' do + before do + account_statuses_cleanup_policy.keep_direct = false + account_statuses_cleanup_policy.keep_pinned = false + account_statuses_cleanup_policy.keep_polls = false + account_statuses_cleanup_policy.keep_media = false + account_statuses_cleanup_policy.keep_self_fav = false + account_statuses_cleanup_policy.keep_self_bookmark = false + end + + it 'does not return the recent toot' do + expect(subject.pluck(:id)).to_not include(recent_status.id) + end + + it 'does not return the unrelated toot' do + expect(subject.pluck(:id)).to_not include(unrelated_status.id) + end + + it 'returns every other old status for deletion' do + expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id) + end + end + + context 'when policy is set to keep every category of toots' do + before do + account_statuses_cleanup_policy.keep_direct = true + account_statuses_cleanup_policy.keep_pinned = true + account_statuses_cleanup_policy.keep_polls = true + account_statuses_cleanup_policy.keep_media = true + account_statuses_cleanup_policy.keep_self_fav = true + account_statuses_cleanup_policy.keep_self_bookmark = true + end + + it 'does not return unrelated old status' do + expect(subject.pluck(:id)).to_not include(unrelated_status.id) + end + + it 'returns only normal statuses for deletion' do + expect(subject.pluck(:id).sort).to eq [very_old_status.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id].sort + end + end + + context 'when policy is to keep statuses with more than 4 boosts' do + before do + account_statuses_cleanup_policy.min_reblogs = 4 + end + + it 'does not return the recent toot' do + expect(subject.pluck(:id)).to_not include(recent_status.id) + end + + it 'does not return the toot reblogged 5 times' do + expect(subject.pluck(:id)).to_not include(reblogged5.id) + end + + it 'does not return the unrelated toot' do + expect(subject.pluck(:id)).to_not include(unrelated_status.id) + end + + it 'returns old statuses not reblogged as much' do + expect(subject.pluck(:id)).to include(very_old_status.id, faved4.id, faved5.id, reblogged4.id) + end + end + + context 'when policy is to keep statuses with more than 4 favs' do + before do + account_statuses_cleanup_policy.min_favs = 4 + end + + it 'does not return the recent toot' do + expect(subject.pluck(:id)).to_not include(recent_status.id) + end + + it 'does not return the toot faved 5 times' do + expect(subject.pluck(:id)).to_not include(faved5.id) + end + + it 'does not return the unrelated toot' do + expect(subject.pluck(:id)).to_not include(unrelated_status.id) + end + + it 'returns old statuses not faved as much' do + expect(subject.pluck(:id)).to include(very_old_status.id, faved4.id, reblogged4.id, reblogged5.id) + end + end + end +end diff --git a/spec/services/account_statuses_cleanup_service_spec.rb b/spec/services/account_statuses_cleanup_service_spec.rb new file mode 100644 index 000000000..257655c41 --- /dev/null +++ b/spec/services/account_statuses_cleanup_service_spec.rb @@ -0,0 +1,101 @@ +require 'rails_helper' + +describe AccountStatusesCleanupService, type: :service do + let(:account) { Fabricate(:account, username: 'alice', domain: nil) } + let(:account_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) } + let!(:unrelated_status) { Fabricate(:status, created_at: 3.years.ago) } + + describe '#call' do + context 'when the account has not posted anything' do + it 'returns 0 deleted toots' do + expect(subject.call(account_policy)).to eq 0 + end + end + + context 'when the account has posted several old statuses' do + let!(:very_old_status) { Fabricate(:status, created_at: 3.years.ago, account: account) } + let!(:old_status) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:another_old_status) { Fabricate(:status, created_at: 1.year.ago, account: account) } + let!(:recent_status) { Fabricate(:status, created_at: 1.day.ago, account: account) } + + context 'given a budget of 1' do + it 'reports 1 deleted toot' do + expect(subject.call(account_policy, 1)).to eq 1 + end + end + + context 'given a normal budget of 10' do + it 'reports 3 deleted statuses' do + expect(subject.call(account_policy, 10)).to eq 3 + end + + it 'records the last deleted id' do + subject.call(account_policy, 10) + expect(account_policy.last_inspected).to eq [old_status.id, another_old_status.id].max + end + + it 'actually deletes the statuses' do + subject.call(account_policy, 10) + expect(Status.find_by(id: [very_old_status.id, old_status.id, another_old_status.id])).to be_nil + end + end + + context 'when called repeatedly with a budget of 2' do + it 'reports 2 then 1 deleted statuses' do + expect(subject.call(account_policy, 2)).to eq 2 + expect(subject.call(account_policy, 2)).to eq 1 + end + + it 'actually deletes the statuses in the expected order' do + subject.call(account_policy, 2) + expect(Status.find_by(id: very_old_status.id)).to be_nil + subject.call(account_policy, 2) + expect(Status.find_by(id: [very_old_status.id, old_status.id, another_old_status.id])).to be_nil + end + end + + context 'when a self-faved toot is unfaved' do + let!(:self_faved) { Fabricate(:status, created_at: 6.months.ago, account: account) } + let!(:favourite) { Fabricate(:favourite, account: account, status: self_faved) } + + it 'deletes it once unfaved' do + expect(subject.call(account_policy, 20)).to eq 3 + expect(Status.find_by(id: self_faved.id)).to_not be_nil + expect(subject.call(account_policy, 20)).to eq 0 + favourite.destroy! + expect(subject.call(account_policy, 20)).to eq 1 + expect(Status.find_by(id: self_faved.id)).to be_nil + end + end + + context 'when there are more un-deletable old toots than the early search cutoff' do + before do + stub_const 'AccountStatusesCleanupPolicy::EARLY_SEARCH_CUTOFF', 5 + # Old statuses that should be cut-off + 10.times do + Fabricate(:status, created_at: 4.years.ago, visibility: :direct, account: account) + end + # New statuses that prevent cut-off id to reach the last status + 10.times do + Fabricate(:status, created_at: 4.seconds.ago, visibility: :direct, account: account) + end + end + + it 'reports 0 deleted statuses then 0 then 3 then 0 again' do + expect(subject.call(account_policy, 10)).to eq 0 + expect(subject.call(account_policy, 10)).to eq 0 + expect(subject.call(account_policy, 10)).to eq 3 + expect(subject.call(account_policy, 10)).to eq 0 + end + + it 'never causes the recorded id to get higher than oldest deletable toot' do + subject.call(account_policy, 10) + subject.call(account_policy, 10) + subject.call(account_policy, 10) + subject.call(account_policy, 10) + expect(account_policy.last_inspected).to be < Mastodon::Snowflake.id_at(account_policy.min_status_age.seconds.ago, with_random: false) + end + end + end + end +end diff --git a/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb new file mode 100644 index 000000000..8f20725c8 --- /dev/null +++ b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb @@ -0,0 +1,127 @@ +require 'rails_helper' + +describe Scheduler::AccountsStatusesCleanupScheduler do + subject { described_class.new } + + let!(:account1) { Fabricate(:account, domain: nil) } + let!(:account2) { Fabricate(:account, domain: nil) } + let!(:account3) { Fabricate(:account, domain: nil) } + let!(:account4) { Fabricate(:account, domain: nil) } + let!(:remote) { Fabricate(:account) } + + let!(:policy1) { Fabricate(:account_statuses_cleanup_policy, account: account1) } + let!(:policy2) { Fabricate(:account_statuses_cleanup_policy, account: account3) } + let!(:policy3) { Fabricate(:account_statuses_cleanup_policy, account: account4, enabled: false) } + + let(:queue_size) { 0 } + let(:queue_latency) { 0 } + let(:process_set_stub) do + [ + { + 'concurrency' => 2, + 'queues' => ['push', 'default'], + }, + ] + end + let(:retry_size) { 0 } + + before do + queue_stub = double + allow(queue_stub).to receive(:size).and_return(queue_size) + allow(queue_stub).to receive(:latency).and_return(queue_latency) + allow(Sidekiq::Queue).to receive(:new).and_return(queue_stub) + allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set_stub) + + sidekiq_stats_stub = double + allow(sidekiq_stats_stub).to receive(:retry_size).and_return(retry_size) + allow(Sidekiq::Stats).to receive(:new).and_return(sidekiq_stats_stub) + + # Create a bunch of old statuses + 10.times do + Fabricate(:status, account: account1, created_at: 3.years.ago) + Fabricate(:status, account: account2, created_at: 3.years.ago) + Fabricate(:status, account: account3, created_at: 3.years.ago) + Fabricate(:status, account: account4, created_at: 3.years.ago) + Fabricate(:status, account: remote, created_at: 3.years.ago) + end + + # Create a bunch of newer statuses + 5.times do + Fabricate(:status, account: account1, created_at: 3.minutes.ago) + Fabricate(:status, account: account2, created_at: 3.minutes.ago) + Fabricate(:status, account: account3, created_at: 3.minutes.ago) + Fabricate(:status, account: account4, created_at: 3.minutes.ago) + Fabricate(:status, account: remote, created_at: 3.minutes.ago) + end + end + + describe '#under_load?' do + context 'when nothing is queued' do + it 'returns false' do + expect(subject.under_load?).to be false + end + end + + context 'when numerous jobs are queued' do + let(:queue_size) { 5 } + let(:queue_latency) { 120 } + + it 'returns true' do + expect(subject.under_load?).to be true + end + end + + context 'when there is a huge amount of jobs to retry' do + let(:retry_size) { 1_000_000 } + + it 'returns true' do + expect(subject.under_load?).to be true + end + end + end + + describe '#get_budget' do + context 'on a single thread' do + let(:process_set_stub) { [ { 'concurrency' => 1, 'queues' => ['push', 'default'] } ] } + + it 'returns a low value' do + expect(subject.compute_budget).to be < 10 + end + end + + context 'on a lot of threads' do + let(:process_set_stub) do + [ + { 'concurrency' => 2, 'queues' => ['push', 'default'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + ] + end + + it 'returns a larger value' do + expect(subject.compute_budget).to be > 10 + end + end + end + + describe '#perform' do + context 'when the budget is lower than the number of toots to delete' do + it 'deletes as many statuses as the given budget' do + expect { subject.perform }.to change { Status.count }.by(-subject.compute_budget) + end + + it 'does not delete from accounts with no cleanup policy' do + expect { subject.perform }.to_not change { account2.statuses.count } + end + + it 'does not delete from accounts with disabled cleanup policies' do + expect { subject.perform }.to_not change { account4.statuses.count } + end + + it 'eventually deletes every deletable toot' do + expect { subject.perform; subject.perform; subject.perform; subject.perform }.to change { Status.count }.by(-20) + end + end + end +end