MetuFSS
/
metu.life
2
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ProcessFeedService refactor

Browse Source
master
Eugen Rochko 8 years ago
parent dbe00a4156
commit 11cbe49ffc
1 changed files with 147 additions and 195 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 300
      app/services/process_feed_service.rb

300
app/services/process_feed_service.rb
Unescape View File

@ -2,257 +2,209 @@ class ProcessFeedService < BaseService
ACTIVITY_NS = 'http://activitystrea.ms/spec/1.0/'.freeze
THREAD_NS = 'http://purl.org/syndication/thread/1.0'.freeze
# Create local statuses from an Atom feed
# @param [String] body Atom feed
# @param [Account] account Account this feed belongs to
# @return [Enumerable] created statuses
def call(body, account)
xml = Nokogiri::XML(body)
update_remote_profile_service.call(xml.at_xpath('/xmlns:feed/xmlns:author'), account) unless xml.at_xpath('/xmlns:feed').nil?
xml.xpath('//xmlns:entry').reverse_each.map { |entry| process_entry(account, entry) }.compact
update_author(xml, account)
process_entries(xml, account)
end
private
def process_entry(account, entry)
return unless [:note, :comment, :activity].include? object_type(entry)
status = Status.find_by(uri: activity_id(entry))
# If we already have a post and the verb is now "delete", we gotta delete it and move on!
if !status.nil? && verb(entry) == :delete
delete_post!(status)
return
def update_author(xml, account)
return if xml.at_xpath('/xmlns:feed').nil?
UpdateRemoteProfileService.new.call(xml.at_xpath('/xmlns:feed/xmlns:author'), account)
end
return unless status.nil?
status = Status.new(uri: activity_id(entry), url: activity_link(entry), account: account, text: content(entry), created_at: published(entry), updated_at: updated(entry))
if verb(entry) == :share
add_reblog!(entry, status)
elsif verb(entry) == :post
if thread_id(entry).nil?
add_post!(entry, status)
else
add_reply!(entry, status)
end
else
return
def process_entries(xml, account)
xml.xpath('//xmlns:entry').reverse_each.map { |entry| ProcessEntry.new.call(entry, account) }.compact
end
# If we added a status, go through accounts it mentions and create respective relations
# Also record all media attachments for the status and for the reblogged status if present
unless status.new_record?
record_remote_mentions(status, entry.xpath('./xmlns:link[@rel="mentioned"]'))
record_remote_mentions(status.reblog, entry.at_xpath('./activity:object', activity: ACTIVITY_NS).xpath('./xmlns:link[@rel="mentioned"]')) if status.reblog?
class ProcessEntry
def call(xml, account)
@account = account
@xml = xml
if status.reblog?
ProcessHashtagsService.new.call(status.reblog, entry.at_xpath('./activity:object', activity: ACTIVITY_NS).xpath('./xmlns:category').map { |category| category['term'] })
else
ProcessHashtagsService.new.call(status, entry.xpath('./xmlns:category').map { |category| category['term'] })
end
return if skip_unsupported_type?
process_attachments(entry, status)
process_attachments(entry.xpath('./activity:object', activity: ACTIVITY_NS), status.reblog) if status.reblog?
Rails.logger.debug "Queuing remote status #{status.id} for distribution"
DistributionWorker.perform_async(status.id)
return status
case verb
when :post, :share
return create_status
when :delete
return delete_status
end
end
def record_remote_mentions(status, links)
return if status.local?
# Here we have to do a reverse lookup of local accounts by their URL!
# It's not pretty at all! I really wish all these protocols sticked to
# using acct:username@domain only! It would make things so much easier
# and tidier
links.each do |mention_link|
href_val = mention_link.attribute('href').value
next if href_val == 'http://activityschema.org/collection/public'
href = Addressable::URI.parse(href_val)
private
if TagManager.instance.local_domain?(href.host)
# A local user is mentioned
mentioned_account = Account.find_local(href.path.gsub('/users/', ''))
def create_status
Rails.logger.debug "Creating remote status #{id}"
status = status_from_xml(@xml)
unless mentioned_account.nil?
mentioned_account.mentions.where(status: status).first_or_create(status: status)
NotificationMailer.mention(mentioned_account, status).deliver_later unless mentioned_account.blocking?(status.account)
if verb == :share
original_status = status_from_xml(xml.at_xpath('.//activity:object', activity: ACTIVITY_NS))
status.reblog = original_status
end
else
# What to do about remote user?
# This is kinda dodgy because URLs could change, we don't index them
mentioned_account = Account.find_by(url: href.to_s)
if mentioned_account.nil?
mentioned_account = FetchRemoteAccountService.new.call(href)
status.save!
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
DistributionWorker.perform_async(status.id)
status
end
unless mentioned_account.nil?
mentioned_account.mentions.where(status: status).first_or_create(status: status)
end
end
def delete_status
Rails.logger.debug "Deleting remote status #{id}"
status = Status.find_by(uri: id)
RemoveStatusService.new.call(status) unless status.nil?
nil
end
def skip_unsupported_type?
!([:post, :share, :delete].include?(verb) && [:activity, :note, :comment].include?(type))
end
def process_attachments(entry, status)
return if status.local?
def status_from_xml(entry)
# Return early if status already exists in db
status = find_status(id(entry))
return status unless status.nil?
entry.xpath('./xmlns:link[@rel="enclosure"]').each do |enclosure_link|
next if enclosure_link.attribute('href').nil?
status = Status.create!({
uri: id(entry),
url: url(entry),
account: account?(entry) ? find_or_resolve_account(acct(entry)) : @account,
text: content(entry),
created_at: published(entry),
})
media = MediaAttachment.where(status: status, remote_url: enclosure_link.attribute('href').value).first
if thread?(entry)
status.thread = find_or_resolve_status(status, *thread(entry))
end
next unless media.nil?
mentions_from_xml(status, entry)
hashtags_from_xml(status, entry)
media_from_xml(status, entry)
begin
media = MediaAttachment.new(account: status.account, status: status, remote_url: enclosure_link.attribute('href').value)
media.file_remote_url = enclosure_link.attribute('href').value
media.save
rescue Paperclip::Errors::NotIdentifiedByImageMagickError
Rails.logger.debug "Error saving attachment from #{enclosure_link.attribute('href').value}"
next
end
end
status
end
def add_post!(_entry, status)
status.save!
def find_or_resolve_account(acct)
FollowRemoteAccountService.new.call(acct)
end
def add_reblog!(entry, status)
status.reblog = find_original_status(entry, target_id(entry))
def find_or_resolve_status(parent, uri, url)
status = find_status(uri)
ThreadResolveWorker.perform_async(parent.id, url) if status.nil?
if status.reblog.nil?
status.reblog = fetch_remote_status(entry)
status
end
if !status.reblog.nil?
status.save!
NotificationMailer.reblog(status.reblog, status.account).deliver_later if status.reblog.local? && !status.reblog.account.blocking?(status.account)
end
def find_status(uri)
if TagManager.instance.local_id?(uri)
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status')
return Status.find(local_id)
end
def add_reply!(entry, status)
status.thread = find_original_status(entry, thread_id(entry))
status.save!
if status.thread.nil? && !thread_href(entry).nil?
ThreadResolveWorker.perform_async(status.id, thread_href(entry))
end
Status.find_by(uri: uri)
end
def delete_post!(status)
remove_status_service.call(status)
end
def mentions_from_xml(parent, xml)
processed_account_ids = []
xml.xpath('./xmlns:link[@rel="mentioned"]').each do |link|
next if link['href'] == 'http://activityschema.org/collection/public'
def find_original_status(_xml, id)
return nil if id.nil?
url = Addressable::URI.parse(link['href'])
if TagManager.instance.local_id?(id)
Status.find(TagManager.instance.unique_tag_to_local_id(id, 'Status'))
mentioned_account = if TagManager.instance.local_domain?(url.host)
Account.find_local(url.path.gsub('/users/', ''))
else
Status.find_by(uri: id)
end
Account.find_by(url: link['href']) || FetchRemoteAccountService.new.call(link['href'])
end
def fetch_remote_status(xml)
username = xml.at_xpath('./activity:object', activity: ACTIVITY_NS).at_xpath('./xmlns:author/xmlns:name').content
url = xml.at_xpath('./activity:object', activity: ACTIVITY_NS).at_xpath('./xmlns:author/xmlns:uri').content
domain = Addressable::URI.parse(url).host
account = Account.find_remote(username, domain)
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
if account.nil?
account = follow_remote_account_service.call("#{username}@#{domain}")
if mentioned_account.local?
# Send notifications
NotificationMailer.mention(mentioned_account, parent).deliver_later unless mentioned_account.blocking?(parent.account)
end
status = Status.new(account: account, uri: target_id(xml), text: target_content(xml), url: target_url(xml), created_at: published(xml), updated_at: updated(xml))
status.thread = find_original_status(xml, thread_id(xml))
mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
if status.save && status.thread.nil? && !thread_href(xml).nil?
ThreadResolveWorker.perform_async(status.id, thread_href(xml))
# So we can skip duplicate mentions
processed_account_ids << mentioned_account.id
end
status
rescue Goldfinger::Error, HTTP::Error
nil
end
def published(xml)
xml.at_xpath('./xmlns:published').content
def hashtags_from_xml(parent, xml)
tags = xml.xpath('./xmlns:category').map { |category| category['term'] }
ProcessHashtagsService.new.call(parent, tags)
end
def updated(xml)
xml.at_xpath('./xmlns:updated').content
end
def media_from_xml(parent, xml)
xml.xpath('./xmlns:link[@rel="enclosure"]').each do |link|
next unless link['href']
def content(xml)
xml.at_xpath('./xmlns:content').try(:content)
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
begin
media.file_remote_url = link['href']
media.save
rescue Paperclip::Errors::NotIdentifiedByImageMagickError
next
end
end
end
def thread_id(xml)
xml.at_xpath('./thr:in-reply-to', thr: THREAD_NS).attribute('ref').value
rescue
nil
def id(xml = @xml)
xml.at_xpath('./xmlns:id').content
end
def thread_href(xml)
xml.at_xpath('./thr:in-reply-to', thr: THREAD_NS).attribute('href').value
def verb(xml = @xml)
raw = xml.at_xpath('./activity:verb', activity: ACTIVITY_NS).content
raw.gsub('http://activitystrea.ms/schema/1.0/', '').gsub('http://ostatus.org/schema/1.0/', '').to_sym
rescue
nil
:post
end
def target_id(xml)
xml.at_xpath('.//activity:object', activity: ACTIVITY_NS).at_xpath('./xmlns:id').content
def type(xml = @xml)
raw = xml.at_xpath('./activity:object-type', activity: ACTIVITY_NS).content
raw.gsub('http://activitystrea.ms/schema/1.0/', '').gsub('http://ostatus.org/schema/1.0/', '').to_sym
rescue
nil
:activity
end
def activity_id(xml)
xml.at_xpath('./xmlns:id').content
def url(xml = @xml)
link = xml.at_xpath('./xmlns:link[@rel="alternate"]')
link['href']
end
def activity_link(xml)
xml.at_xpath('./xmlns:link[@rel="alternate"]').attribute('href').value
rescue
''
def content(xml = @xml)
xml.at_xpath('./xmlns:content').content
end
def target_content(xml)
xml.at_xpath('.//activity:object', activity: ACTIVITY_NS).at_xpath('./xmlns:content').content
def published(xml = @xml)
xml.at_xpath('./xmlns:published').content
end
def target_url(xml)
xml.at_xpath('.//activity:object', activity: ACTIVITY_NS).at_xpath('./xmlns:link[@rel="alternate"]').attribute('href').value
def thread?(xml = @xml)
!xml.at_xpath('./thr:in-reply-to', thr: THREAD_NS).nil?
end
def object_type(xml)
xml.at_xpath('./activity:object-type', activity: ACTIVITY_NS).content.gsub('http://activitystrea.ms/schema/1.0/', '').gsub('http://ostatus.org/schema/1.0/', '').to_sym
rescue
:activity
def thread(xml = @xml)
thr = xml.at_xpath('./thr:in-reply-to', thr: THREAD_NS)
[thr['ref'], thr['href']]
end
def verb(xml)
xml.at_xpath('./activity:verb', activity: ACTIVITY_NS).content.gsub('http://activitystrea.ms/schema/1.0/', '').gsub('http://ostatus.org/schema/1.0/', '').to_sym
rescue
:post
def account?(xml = @xml)
!xml.at_xpath('./xmlns:author').nil?
end
def follow_remote_account_service
@follow_remote_account_service ||= FollowRemoteAccountService.new
end
def acct(xml = @xml)
username = xml.at_xpath('./xmlns:author/xmlns:name').content
url = xml.at_xpath('./xmlns:author/xmlns:uri').content
domain = Addressable::URI.parse(url).host
def update_remote_profile_service
@update_remote_profile_service ||= UpdateRemoteProfileService.new
"#{username}@#{domain}"
end
def remove_status_service
@remove_status_service ||= RemoveStatusService.new
end
end

Write Preview
Loading…
Cancel
Save