MetuFSS
/
metu.life
2
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Change `tootctl media remove-orphans` to work for all classes (#13316)

Browse Source 
Change `tootctl media lookup` to not use an interactive prompt
master
Eugen Rochko 5 years ago committed by GitHub
parent e187537dfd
commit 0c8945e5ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      app/lib/activitypub/tag_manager.rb
  2. 13
      app/models/media_attachment.rb
  3. 107
      lib/mastodon/media_cli.rb
  4. 13
      lib/paperclip/attachment_extensions.rb

2
app/lib/activitypub/tag_manager.rb
Unescape View File

@ -15,6 +15,8 @@ class ActivityPub::TagManager
def url_for(target)
return target.url if target.respond_to?(:local?) && !target.local?
return unless target.respond_to?(:object_type)
case target.object_type
when :person
target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)

13
app/models/media_attachment.rb
Unescape View File

@ -184,19 +184,6 @@ class MediaAttachment < ApplicationRecord
audio? || video?
end
def variant?(other_file_name)
return true if file_file_name == other_file_name
return false if file_file_name.nil?
formats = file.styles.values.map(&:format).compact
return false if formats.empty?
extension = File.extname(other_file_name)
formats.include?(extension.delete('.')) && File.basename(other_file_name, extension) == File.basename(file_file_name, File.extname(file_file_name))
end
def to_param
shortcode
end

107
lib/mastodon/media_cli.rb
Unescape View File

@ -45,6 +45,7 @@ module Mastodon
end
option :start_after
option :prefix
option :dry_run, type: :boolean, default: false
desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
long_desc <<~LONG_DESC
@ -58,6 +59,7 @@ module Mastodon
reclaimed_bytes = 0
removed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
prefix = options[:prefix]
case Paperclip::Attachment.default_options[:storage]
when :s3
@ -69,7 +71,7 @@ module Mastodon
loop do
objects = begin
begin
bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x }
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
@ -80,15 +82,20 @@ module Mastodon
break if objects.empty?
last_key = objects.last.key
attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment }
record_map = preload_records_from_mixed_objects(objects)
objects.each do |object|
attachment_id = object.key.split('/')[2..-2].join.to_i
filename = object.key.split('/').last
path_segments = object.key.split('/')
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i
file_name = path_segments.last
record = record_map.dig(model_name, record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename)
next unless attachment.blank? || !attachment.variant?(file_name)
begin
object.delete unless options[:dry_run]
@ -110,17 +117,24 @@ module Mastodon
root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
Find.find(File.join(root_path, 'media_attachments', 'files')) do |path|
Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i
filename = key.split(File::SEPARATOR).last
attachment = MediaAttachment.find_by(id: attachment_id)
path_segments = key.split(File::SEPARATOR)
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
record = model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.nil? || !attachment.variant?(filename)
next unless attachment.blank? || !attachment.variant?(file_name)
begin
size = File.size(path)
@ -213,25 +227,66 @@ module Mastodon
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
end
desc 'lookup', 'Lookup where media is displayed by passing a media URL'
def lookup
prompt = TTY::Prompt.new
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url)
path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..-1]
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
url = prompt.ask('Please enter a URL to the media to lookup:', required: true)
unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red)
exit(1)
end
attachment_id = url
.split('/')[0..-2]
.grep(/\A\d+\z/)
.join('')
record = model_name.constantize.find_by(id: record_id)
record = record.status if record.respond_to?(:status)
if url.split('/')[0..-2].include? 'media_attachments'
model = MediaAttachment.find(attachment_id).status
prompt.say(ActivityPub::TagManager.instance.url_for(model))
elsif url.split('/')[0..-2].include? 'accounts'
model = Account.find(attachment_id)
prompt.say(ActivityPub::TagManager.instance.url_for(model))
else
prompt.say('Not found')
unless record
say('Cannot find corresponding record', :red)
exit(1)
end
display_url = ActivityPub::TagManager.instance.url_for(record)
if display_url.blank?
say('No public URL for this type of record', :red)
exit(1)
end
say(display_url, :blue)
rescue Addressable::URI::InvalidURIError
say('Invalid URL', :red)
exit(1)
end
private
PRELOAD_MODEL_WHITELIST = %w(
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload
).freeze
def preload_records_from_mixed_objects(objects)
preload_map = Hash.new { |hash, key| hash[key] = [] }
objects.map do |object|
segments = object.key.split('/').first
model_name = segments.first.classify
record_id = segments[2..-2].join.to_i
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
preload_map[model_name] << record_id
end
preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
model_map[model_name] = model_name.constantize.where(id: record_ids).each_with_object({}) { |record, record_map| record_map[record.id] = record }
end
end
end

13
lib/paperclip/attachment_extensions.rb
Unescape View File

@ -24,6 +24,19 @@ module Paperclip
flush_deletes
end
end
def variant?(other_filename)
return true if original_filename == other_filename
return false if original_filename.nil?
formats = styles.values.map(&:format).compact
return false if formats.empty?
other_extension = File.extname(other_filename)
formats.include?(other_extension.delete('.')) && File.basename(other_filename, other_extension) == File.basename(original_filename, File.extname(original_filename))
end
end
end

Write Preview
Loading…
Cancel
Save