Merge branch 'main' into glitch-soc/merge-upstream

master
Claire 1 year ago
commit 2b4c7beb73
  1. 12
      .circleci/config.yml
  2. 12
      .github/workflows/build-image.yml
  3. 2
      .github/workflows/check-i18n.yml
  4. 2
      .ruby-version
  5. 17
      Dockerfile
  6. 15
      Gemfile
  7. 106
      Gemfile.lock
  8. 24
      app/chewy/accounts_index.rb
  9. 48
      app/chewy/statuses_index.rb
  10. 16
      app/chewy/tags_index.rb
  11. 1
      app/javascript/styles/mastodon/admin.scss
  12. 2
      app/javascript/styles/mastodon/emoji_picker.scss
  13. 4
      app/lib/link_details_extractor.rb
  14. 4
      app/lib/webfinger.rb
  15. 2
      app/models/account.rb
  16. 2
      app/models/account_stat.rb
  17. 2
      app/models/bookmark.rb
  18. 2
      app/models/favourite.rb
  19. 2
      app/models/status.rb
  20. 2
      app/models/tag.rb
  21. 2
      app/services/batched_remove_status_service.rb
  22. 4
      app/services/delete_account_service.rb
  23. 20
      config/initializers/chewy.rb
  24. 8
      docker-compose.yml
  25. 99
      lib/mastodon/search_cli.rb
  26. 2
      lib/paperclip/response_with_limit_adapter.rb
  27. 18
      package.json
  28. 224
      yarn.lock

@ -23,7 +23,7 @@ executors:
environment:
POSTGRES_USER: root
POSTGRES_HOST_AUTH_METHOD: trust
- image: circleci/redis:5-alpine
- image: circleci/redis:6-alpine
commands:
install-system-dependencies:
@ -45,7 +45,7 @@ commands:
bundle config without 'development production'
name: Set bundler settings
- ruby/install-deps:
bundler-version: '2.2.29'
bundler-version: '2.2.31'
key: ruby<< parameters.ruby-version >>-gems-v1
wait-db:
steps:
@ -56,14 +56,14 @@ commands:
jobs:
build:
docker:
- image: cimg/ruby:2.7-node
- image: cimg/ruby:3.0-node
environment:
RAILS_ENV: test
steps:
- checkout
- install-system-dependencies
- install-ruby-dependencies:
ruby-version: '2.7'
ruby-version: '3.0'
- node/install-packages:
cache-version: v1
pkg-manager: yarn
@ -111,12 +111,12 @@ jobs:
test-migrations:
executor:
name: default
ruby-version: '2.7'
ruby-version: '3.0'
steps:
- checkout
- install-system-dependencies
- install-ruby-dependencies:
ruby-version: '2.7'
ruby-version: '3.0'
- wait-db
- run:
command: ./bin/rails db:create

@ -1,5 +1,6 @@
name: Build container image
on:
workflow_dispatch:
push:
branches:
- "main"
@ -13,15 +14,14 @@ jobs:
- uses: docker/setup-buildx-action@v1
- uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- uses: docker/metadata-action@v3
id: meta
with:
images: ghcr.io/${{ github.repository_owner }}/mastodon
images: tootsuite/mastodon
flavor: |
latest=true
latest=auto
tags: |
type=edge,branch=main
type=semver,pattern={{ raw }}
@ -30,5 +30,5 @@ jobs:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest
cache-from: type=registry,ref=tootsuite/mastodon:latest
cache-to: type=inline

@ -22,7 +22,7 @@ jobs:
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '2.7'
ruby-version: '3.0'
bundler-cache: true
- name: Check locale file normalization
run: bundle exec i18n-tasks check-normalized

@ -1 +1 @@
2.7.4
3.0.2

@ -2,9 +2,10 @@ FROM ubuntu:20.04 as build-dep
# Use bash for the shell
SHELL ["/bin/bash", "-c"]
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
# Install Node v14 (LTS)
ENV NODE_VER="14.17.6"
# Install Node v16 (LTS)
ENV NODE_VER="16.13.0"
RUN ARCH= && \
dpkgArch="$(dpkg --print-architecture)" && \
case "${dpkgArch##*-}" in \
@ -18,15 +19,15 @@ RUN ARCH= && \
esac && \
echo "Etc/UTC" > /etc/localtime && \
apt-get update && \
apt-get install -y --no-install-recommends ca-certificates wget python && \
apt-get install -y --no-install-recommends ca-certificates wget python apt-utils && \
cd ~ && \
wget -q https://nodejs.org/download/release/v$NODE_VER/node-v$NODE_VER-linux-$ARCH.tar.gz && \
tar xf node-v$NODE_VER-linux-$ARCH.tar.gz && \
rm node-v$NODE_VER-linux-$ARCH.tar.gz && \
mv node-v$NODE_VER-linux-$ARCH /opt/node
# Install Ruby
ENV RUBY_VER="2.7.4"
# Install Ruby 3.0
ENV RUBY_VER="3.0.2"
RUN apt-get update && \
apt-get install -y --no-install-recommends build-essential \
bison libyaml-dev libgdbm-dev libreadline-dev libjemalloc-dev \
@ -45,7 +46,8 @@ RUN apt-get update && \
ENV PATH="${PATH}:/opt/ruby/bin:/opt/node/bin"
RUN npm install -g yarn && \
RUN npm install -g npm@latest && \
npm install -g yarn && \
gem install bundler && \
apt-get update && \
apt-get install -y --no-install-recommends git libicu-dev libidn11-dev \
@ -82,11 +84,12 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
# Install mastodon runtime deps
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
RUN apt-get update && \
apt-get -y --no-install-recommends install \
libssl1.1 libpq5 imagemagick ffmpeg libjemalloc2 \
libicu66 libprotobuf17 libidn11 libyaml-0-2 \
file ca-certificates tzdata libreadline8 gcc tini && \
file ca-certificates tzdata libreadline8 gcc tini apt-utils && \
ln -s /opt/mastodon /mastodon && \
gem install bundler && \
rm -rf /var/cache && \

@ -4,6 +4,7 @@ source 'https://rubygems.org'
ruby '>= 2.5.0', '< 3.1.0'
gem 'pkg-config', '~> 1.4'
gem 'rexml', '~> 3.2'
gem 'puma', '~> 5.5'
gem 'rails', '~> 6.1.4'
@ -17,7 +18,7 @@ gem 'makara', '~> 0.5'
gem 'pghero', '~> 2.8'
gem 'dotenv-rails', '~> 2.7'
gem 'aws-sdk-s3', '~> 1.104', require: false
gem 'aws-sdk-s3', '~> 1.106', require: false
gem 'fog-core', '<= 2.1.0'
gem 'fog-openstack', '~> 0.3', require: false
gem 'kt-paperclip', '~> 7.0'
@ -29,7 +30,7 @@ gem 'bootsnap', '~> 1.9.1', require: false
gem 'browser'
gem 'charlock_holmes', '~> 0.7.7'
gem 'iso-639'
gem 'chewy', '~> 5.2'
gem 'chewy', '~> 7.2'
gem 'cld3', '~> 3.4.2'
gem 'devise', '~> 4.8'
gem 'devise-two-factor', '~> 4.0'
@ -59,7 +60,7 @@ gem 'httplog', '~> 1.5.0'
gem 'idn-ruby', require: 'idn'
gem 'kaminari', '~> 1.2'
gem 'link_header', '~> 0.0'
gem 'mime-types', '~> 3.3.1', require: 'mime/types/columnar'
gem 'mime-types', '~> 3.4.1', require: 'mime/types/columnar'
gem 'nokogiri', '~> 1.12'
gem 'nsa', '~> 0.2'
gem 'oj', '~> 3.13'
@ -78,13 +79,13 @@ gem 'rqrcode', '~> 2.1'
gem 'ruby-progressbar', '~> 1.11'
gem 'sanitize', '~> 6.0'
gem 'scenic', '~> 1.5'
gem 'sidekiq', '~> 6.2'
gem 'sidekiq', '~> 6.3'
gem 'sidekiq-scheduler', '~> 3.1'
gem 'sidekiq-unique-jobs', '~> 7.1'
gem 'sidekiq-bulk', '~>0.2.0'
gem 'simple-navigation', '~> 4.3'
gem 'simple_form', '~> 5.1'
gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'
gem 'sprockets-rails', '~> 3.4', require: 'sprockets/railtie'
gem 'stoplight', '~> 2.2.1'
gem 'strong_migrations', '~> 0.7'
gem 'tty-prompt', '~> 0.23', require: false
@ -132,9 +133,9 @@ group :development do
gem 'binding_of_caller', '~> 1.0'
gem 'bullet', '~> 6.1'
gem 'letter_opener', '~> 1.7'
gem 'letter_opener_web', '~> 1.4'
gem 'letter_opener_web', '~> 2.0'
gem 'memory_profiler'
gem 'rubocop', '~> 1.22', require: false
gem 'rubocop', '~> 1.23', require: false
gem 'rubocop-rails', '~> 2.12', require: false
gem 'brakeman', '~> 5.1', require: false
gem 'bundler-audit', '~> 0.9', require: false

@ -79,17 +79,17 @@ GEM
encryptor (~> 3.0.0)
awrence (1.1.1)
aws-eventstream (1.2.0)
aws-partitions (1.519.0)
aws-sdk-core (3.121.3)
aws-partitions (1.532.0)
aws-sdk-core (3.122.1)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.239.0)
aws-partitions (~> 1, >= 1.525.0)
aws-sigv4 (~> 1.1)
jmespath (~> 1.0)
aws-sdk-kms (1.50.0)
aws-sdk-core (~> 3, >= 3.121.2)
aws-sdk-kms (1.51.0)
aws-sdk-core (~> 3, >= 3.122.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.104.0)
aws-sdk-core (~> 3, >= 3.121.2)
aws-sdk-s3 (1.106.0)
aws-sdk-core (~> 3, >= 3.122.0)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.4)
aws-sigv4 (1.4.0)
@ -147,9 +147,9 @@ GEM
activesupport
cbor (0.5.9.6)
charlock_holmes (0.7.7)
chewy (5.2.0)
chewy (7.2.3)
activesupport (>= 5.2)
elasticsearch (>= 2.0.0)
elasticsearch (>= 7.12.0, < 7.14.0)
elasticsearch-dsl
chunky_png (1.4.0)
cld3 (3.4.2)
@ -197,13 +197,13 @@ GEM
railties (>= 3.2)
e2mmap (0.1.0)
ed25519 (1.2.4)
elasticsearch (7.10.1)
elasticsearch-api (= 7.10.1)
elasticsearch-transport (= 7.10.1)
elasticsearch-api (7.10.1)
elasticsearch (7.13.3)
elasticsearch-api (= 7.13.3)
elasticsearch-transport (= 7.13.3)
elasticsearch-api (7.13.3)
multi_json
elasticsearch-dsl (0.1.9)
elasticsearch-transport (7.10.1)
elasticsearch-dsl (0.1.10)
elasticsearch-transport (7.13.3)
faraday (~> 1)
multi_json
encryptor (3.0.0)
@ -214,11 +214,25 @@ GEM
fabrication (2.22.0)
faker (2.19.0)
i18n (>= 1.6, < 2)
faraday (1.3.0)
faraday (1.8.0)
faraday-em_http (~> 1.0)
faraday-em_synchrony (~> 1.0)
faraday-excon (~> 1.1)
faraday-httpclient (~> 1.0.1)
faraday-net_http (~> 1.0)
faraday-net_http_persistent (~> 1.1)
faraday-patron (~> 1.0)
faraday-rack (~> 1.0)
multipart-post (>= 1.2, < 3)
ruby2_keywords
ruby2_keywords (>= 0.0.4)
faraday-em_http (1.0.0)
faraday-em_synchrony (1.0.0)
faraday-excon (1.1.0)
faraday-httpclient (1.0.1)
faraday-net_http (1.0.1)
faraday-net_http_persistent (1.2.0)
faraday-patron (1.0.0)
faraday-rack (1.0.0)
fast_blank (1.0.1)
fastimage (2.2.5)
ffi (1.15.4)
@ -275,9 +289,9 @@ GEM
httplog (1.5.0)
rack (>= 1.0)
rainbow (>= 2.0.0)
i18n (1.8.10)
i18n (1.8.11)
concurrent-ruby (~> 1.0)
i18n-tasks (0.9.34)
i18n-tasks (0.9.35)
activesupport (>= 4.0.2)
ast (>= 2.1.0)
erubi
@ -287,7 +301,7 @@ GEM
rails-i18n
rainbow (>= 2.2.2, < 4.0)
terminal-table (>= 1.5.1)
idn-ruby (0.1.2)
idn-ruby (0.1.4)
ipaddress (0.8.3)
iso-639 (0.3.5)
jmespath (1.4.0)
@ -327,10 +341,11 @@ GEM
addressable (~> 2.7)
letter_opener (1.7.0)
launchy (~> 2.2)
letter_opener_web (1.4.1)
actionmailer (>= 3.2)
letter_opener (~> 1.0)
railties (>= 3.2)
letter_opener_web (2.0.0)
actionmailer (>= 5.2)
letter_opener (~> 1.7)
railties (>= 5.2)
rexml
link_header (0.0.8)
llhttp-ffi (0.4.0)
ffi-compiler (~> 1.0)
@ -356,9 +371,9 @@ GEM
microformats (4.3.1)
json (~> 2.2)
nokogiri (~> 1.10)
mime-types (3.3.1)
mime-types (3.4.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2021.0901)
mime-types-data (3.2021.1115)
mini_mime (1.1.2)
mini_portile2 (2.6.1)
minitest (5.14.4)
@ -395,7 +410,7 @@ GEM
openssl (2.2.0)
openssl-signature_algorithm (0.4.0)
orm_adapter (0.5.0)
ox (2.14.5)
ox (2.14.6)
parallel (1.21.0)
parser (3.0.2.0)
ast (~> 2.4.1)
@ -520,7 +535,7 @@ GEM
rspec-support (3.10.2)
rspec_junit_formatter (0.4.1)
rspec-core (>= 2, < 4, != 2.12.0)
rubocop (1.22.3)
rubocop (1.23.0)
parallel (~> 1.10)
parser (>= 3.0.0.0)
rainbow (>= 2.2.2, < 4.0)
@ -529,7 +544,7 @@ GEM
rubocop-ast (>= 1.12.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.12.0)
rubocop-ast (1.13.0)
parser (>= 3.0.1.1)
rubocop-rails (2.12.4)
activesupport (>= 4.2.0)
@ -539,7 +554,7 @@ GEM
ruby-saml (1.13.0)
nokogiri (>= 1.10.5)
rexml
ruby2_keywords (0.0.4)
ruby2_keywords (0.0.5)
rufus-scheduler (3.7.0)
fugit (~> 1.1, >= 1.1.6)
safety_net_attestation (0.4.0)
@ -552,7 +567,7 @@ GEM
railties (>= 4.0.0)
securecompare (1.0.0)
semantic_range (3.0.0)
sidekiq (6.2.2)
sidekiq (6.3.1)
connection_pool (>= 2.2.2)
rack (~> 2.0)
redis (>= 4.2.0)
@ -584,9 +599,9 @@ GEM
sprockets (3.7.2)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.2)
actionpack (>= 4.0)
activesupport (>= 4.0)
sprockets-rails (3.4.0)
actionpack (>= 5.2)
activesupport (>= 5.2)
sprockets (>= 3.0.0)
sshkit (1.21.2)
net-scp (>= 1.1.2)
@ -597,8 +612,8 @@ GEM
strong_migrations (0.7.8)
activerecord (>= 5)
temple (0.8.2)
terminal-table (3.0.0)
unicode-display_width (~> 1.1, >= 1.1.1)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
terrapin (0.6.0)
climate_control (>= 0.0.3, < 1.0)
thor (1.1.0)
@ -628,7 +643,7 @@ GEM
unf (0.1.4)
unf_ext
unf_ext (0.0.8)
unicode-display_width (1.8.0)
unicode-display_width (2.1.0)
uniform_notifier (1.14.2)
warden (1.2.9)
rack (>= 2.0.9)
@ -661,7 +676,7 @@ GEM
xorcist (1.1.2)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.4.2)
zeitwerk (2.5.1)
PLATFORMS
ruby
@ -671,7 +686,7 @@ DEPENDENCIES
active_record_query_trace (~> 1.8)
addressable (~> 2.8)
annotate (~> 3.1)
aws-sdk-s3 (~> 1.104)
aws-sdk-s3 (~> 1.106)
better_errors (~> 2.9)
binding_of_caller (~> 1.0)
blurhash (~> 0.1)
@ -686,7 +701,7 @@ DEPENDENCIES
capistrano-yarn (~> 2.0)
capybara (~> 3.36)
charlock_holmes (~> 0.7.7)
chewy (~> 5.2)
chewy (~> 7.2)
cld3 (~> 3.4.2)
climate_control (~> 0.2)
color_diff (~> 0.1)
@ -720,14 +735,14 @@ DEPENDENCIES
kaminari (~> 1.2)
kt-paperclip (~> 7.0)
letter_opener (~> 1.7)
letter_opener_web (~> 1.4)
letter_opener_web (~> 2.0)
link_header (~> 0.0)
lograge (~> 0.11)
makara (~> 0.5)
mario-redis-lock (~> 1.2)
memory_profiler
microformats (~> 4.2)
mime-types (~> 3.3.1)
mime-types (~> 3.4.1)
net-ldap (~> 0.17)
nokogiri (~> 1.12)
nsa (~> 0.2)
@ -759,16 +774,17 @@ DEPENDENCIES
redcarpet (~> 3.5)
redis (~> 4.5)
redis-namespace (~> 1.8)
rexml (~> 3.2)
rqrcode (~> 2.1)
rspec-rails (~> 5.0)
rspec-sidekiq (~> 3.1)
rspec_junit_formatter (~> 0.4)
rubocop (~> 1.22)
rubocop (~> 1.23)
rubocop-rails (~> 2.12)
ruby-progressbar (~> 1.11)
sanitize (~> 6.0)
scenic (~> 1.5)
sidekiq (~> 6.2)
sidekiq (~> 6.3)
sidekiq-bulk (~> 0.2.0)
sidekiq-scheduler (~> 3.1)
sidekiq-unique-jobs (~> 7.1)
@ -776,7 +792,7 @@ DEPENDENCIES
simple_form (~> 5.1)
simplecov (~> 0.21)
sprockets (~> 3.7.2)
sprockets-rails (~> 3.2)
sprockets-rails (~> 3.4)
stackprof
stoplight (~> 2.2.1)
strong_migrations (~> 0.7)

@ -23,21 +23,21 @@ class AccountsIndex < Chewy::Index
},
}
define_type ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } do
root date_detection: false do
field :id, type: 'long'
index_scope ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? }
field :display_name, type: 'text', analyzer: 'content' do
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
end
root date_detection: false do
field :id, type: 'long'
field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
end
field :display_name, type: 'text', analyzer: 'content' do
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
end
field :following_count, type: 'long', value: ->(account) { account.following.local.count }
field :followers_count, type: 'long', value: ->(account) { account.followers.local.count }
field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
end
field :following_count, type: 'long', value: ->(account) { account.following.local.count }
field :followers_count, type: 'long', value: ->(account) { account.followers.local.count }
field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
end
end

@ -31,36 +31,36 @@ class StatusesIndex < Chewy::Index
},
}
define_type ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll) do
crutch :mentions do |collection|
data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll)
crutch :favourites do |collection|
data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
crutch :mentions do |collection|
data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
crutch :reblogs do |collection|
data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
crutch :favourites do |collection|
data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
crutch :bookmarks do |collection|
data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
crutch :reblogs do |collection|
data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
end
root date_detection: false do
field :id, type: 'long'