Browse Source

Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- `.github/ISSUE_TEMPLATE/bug_report.md`:
  Upstream added the `bug` label to bug reports.
  Did the same.
- `app/services/fan_out_on_write_service.rb`:
  Upstream put DMs back into timelines, glitch-soc was already doing it.
  Ignored upstream changes.
master
Thibaut Girka 5 months ago
parent
commit
48f0f3ffee
42 changed files with 1362 additions and 320 deletions
  1. +2
    -2
      .github/ISSUE_TEMPLATE/bug_report.md
  2. +0
    -1
      .github/ISSUE_TEMPLATE/feature_request.md
  3. +1
    -1
      .ruby-version
  4. +1
    -1
      Dockerfile
  5. +3
    -6
      Gemfile
  6. +14
    -16
      Gemfile.lock
  7. +19
    -0
      app/controllers/settings/exports/bookmarks_controller.rb
  8. +7
    -0
      app/javascript/mastodon/actions/app.js
  9. +9
    -6
      app/javascript/mastodon/components/status.js
  10. +5
    -1
      app/javascript/mastodon/containers/status_container.js
  11. +50
    -1
      app/javascript/mastodon/features/audio/index.js
  12. +1
    -1
      app/javascript/mastodon/features/getting_started/components/announcements.js
  13. +37
    -54
      app/javascript/mastodon/features/ui/index.js
  14. +78
    -0
      app/javascript/mastodon/features/video/index.js
  15. +15
    -10
      app/javascript/mastodon/is_mobile.js
  16. +6
    -1
      app/javascript/mastodon/reducers/meta.js
  17. +41
    -9
      app/javascript/mastodon/utils/resize_image.js
  18. +1
    -1
      app/lib/activitypub/activity/delete.rb
  19. +28
    -0
      app/lib/cache_buster.rb
  20. +12
    -0
      app/models/export.rb
  21. +1
    -1
      app/models/import.rb
  22. +4
    -1
      app/services/delete_account_service.rb
  23. +45
    -0
      app/services/import_service.rb
  24. +16
    -9
      app/services/resolve_account_service.rb
  25. +2
    -0
      app/services/suspend_account_service.rb
  26. +2
    -0
      app/services/unsuspend_account_service.rb
  27. +4
    -0
      app/views/settings/exports/show.html.haml
  28. +2
    -1
      app/workers/account_deletion_worker.rb
  29. +18
    -0
      app/workers/cache_buster_worker.rb
  30. +10
    -0
      config/initializers/cache_buster.rb
  31. +0
    -1
      config/initializers/paperclip.rb
  32. +2
    -0
      config/locales/en.yml
  33. +1
    -0
      config/routes.rb
  34. +4
    -0
      lib/cli.rb
  35. +610
    -0
      lib/mastodon/maintenance_cli.rb
  36. +11
    -0
      lib/tasks/db.rake
  37. +9
    -9
      package.json
  38. +17
    -0
      spec/controllers/settings/exports/bookmarks_controller_specs.rb
  39. +4
    -0
      spec/fixtures/files/bookmark-imports.txt
  40. +42
    -0
      spec/services/import_service_spec.rb
  41. +45
    -7
      spec/services/resolve_account_service_spec.rb
  42. +183
    -180
      yarn.lock

+ 2
- 2
.github/ISSUE_TEMPLATE/bug_report.md View File

@@ -1,7 +1,7 @@
---
name: Bug Report
about: Create a report to help us improve
about: If something isn't working as expected
labels: bug
---

[Issue text goes here].


+ 0
- 1
.github/ISSUE_TEMPLATE/feature_request.md View File

@@ -1,7 +1,6 @@
---
name: Feature Request
about: I have a suggestion

---

<!-- Please use a concise and distinct title for the issue -->


+ 1
- 1
.ruby-version View File

@@ -1 +1 @@
2.6.6
2.7.2

+ 1
- 1
Dockerfile View File

@@ -40,7 +40,7 @@ RUN apt update && \
cd .. && rm -rf jemalloc-$JE_VER $JE_VER.tar.gz

# Install Ruby
ENV RUBY_VER="2.6.6"
ENV RUBY_VER="2.7.2"
ENV CPPFLAGS="-I/opt/jemalloc/include"
ENV LDFLAGS="-L/opt/jemalloc/lib/"
RUN apt update && \


+ 3
- 6
Gemfile View File

@@ -11,9 +11,6 @@ gem 'sprockets', '~> 3.7.2'
gem 'thor', '~> 1.0'
gem 'rack', '~> 2.2.3'

gem 'thwait', '~> 0.2.0'
gem 'e2mmap', '~> 0.1.0'

gem 'hamlit-rails', '~> 0.2'
gem 'pg', '~> 1.2'
gem 'makara', '~> 0.4'
@@ -44,7 +41,7 @@ group :pam_authentication, optional: true do
end

gem 'net-ldap', '~> 0.16'
gem 'omniauth-cas', '~> 1.1'
gem 'omniauth-cas', '~> 2.0'
gem 'omniauth-saml', '~> 1.10'
gem 'omniauth', '~> 1.9'

@@ -127,7 +124,7 @@ group :test do
gem 'rails-controller-testing', '~> 1.0'
gem 'rspec-sidekiq', '~> 3.1'
gem 'simplecov', '~> 0.19', require: false
gem 'webmock', '~> 3.9'
gem 'webmock', '~> 3.10'
gem 'parallel_tests', '~> 3.3'
gem 'rspec_junit_formatter', '~> 0.4'
end
@@ -141,7 +138,7 @@ group :development do
gem 'letter_opener', '~> 1.7'
gem 'letter_opener_web', '~> 1.4'
gem 'memory_profiler'
gem 'rubocop', '~> 0.93', require: false
gem 'rubocop', '~> 1.3', require: false
gem 'rubocop-rails', '~> 2.8', require: false
gem 'brakeman', '~> 4.10', require: false
gem 'bundler-audit', '~> 0.7', require: false


+ 14
- 16
Gemfile.lock View File

@@ -79,7 +79,7 @@ GEM
cocaine (~> 0.5.3)
awrence (1.1.1)
aws-eventstream (1.1.0)
aws-partitions (1.390.0)
aws-partitions (1.393.0)
aws-sdk-core (3.109.2)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.239.0)
@@ -88,7 +88,7 @@ GEM
aws-sdk-kms (1.39.0)
aws-sdk-core (~> 3, >= 3.109.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.84.0)
aws-sdk-s3 (1.84.1)
aws-sdk-core (~> 3, >= 3.109.0)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.1)
@@ -104,7 +104,7 @@ GEM
debug_inspector (>= 0.0.1)
blurhash (0.1.4)
ffi (~> 1.10.0)
bootsnap (1.5.0)
bootsnap (1.5.1)
msgpack (~> 1.0)
brakeman (4.10.0)
browser (4.2.0)
@@ -289,7 +289,7 @@ GEM
jmespath (1.4.0)
json (2.3.1)
json-canonicalization (0.2.0)
json-ld (3.1.4)
json-ld (3.1.5)
htmlentities (~> 4.3)
json-canonicalization (~> 0.2)
link_header (~> 0.0, >= 0.0.8)
@@ -367,11 +367,11 @@ GEM
concurrent-ruby (~> 1.0, >= 1.0.2)
sidekiq (>= 3.5)
statsd-ruby (~> 1.4, >= 1.4.0)
oj (3.10.15)
oj (3.10.16)
omniauth (1.9.1)
hashie (>= 3.4.6)
rack (>= 1.6.2, < 3)
omniauth-cas (1.1.1)
omniauth-cas (2.0.0)
addressable (~> 2.3)
nokogiri (~> 1.5)
omniauth (~> 1.2)
@@ -473,7 +473,7 @@ GEM
thor (>= 0.19.0, < 2.0)
rainbow (3.0.0)
rake (13.0.1)
rdf (3.1.6)
rdf (3.1.7)
hamster (~> 3.0)
link_header (~> 0.0, >= 0.0.8)
rdf-normalize (0.4.0)
@@ -533,16 +533,16 @@ GEM
rspec-support (3.9.3)
rspec_junit_formatter (0.4.1)
rspec-core (>= 2, < 4, != 2.12.0)
rubocop (0.93.1)
rubocop (1.3.0)
parallel (~> 1.10)
parser (>= 2.7.1.5)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8)
rexml
rubocop-ast (>= 0.6.0)
rubocop-ast (>= 1.1.1)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 2.0)
rubocop-ast (0.8.0)
rubocop-ast (1.1.1)
parser (>= 2.7.1.5)
rubocop-rails (2.8.1)
activesupport (>= 4.2.0)
@@ -650,7 +650,7 @@ GEM
safety_net_attestation (~> 0.4.0)
securecompare (~> 1.0)
tpm-key_attestation (~> 0.9.0)
webmock (3.9.5)
webmock (3.10.0)
addressable (>= 2.3.6)
crack (>= 0.3.2)
hashdiff (>= 0.4.0, < 2.0.0)
@@ -705,7 +705,6 @@ DEPENDENCIES
discard (~> 1.2)
doorkeeper (~> 5.4)
dotenv-rails (~> 2.7)
e2mmap (~> 0.1.0)
ed25519 (~> 1.2)
fabrication (~> 2.21)
faker (~> 2.14)
@@ -742,7 +741,7 @@ DEPENDENCIES
nsa (~> 0.2)
oj (~> 3.10)
omniauth (~> 1.9)
omniauth-cas (~> 1.1)
omniauth-cas (~> 2.0)
omniauth-saml (~> 1.10)
ox (~> 2.13)
paperclip (~> 6.0)
@@ -777,7 +776,7 @@ DEPENDENCIES
rspec-rails (~> 4.0)
rspec-sidekiq (~> 3.1)
rspec_junit_formatter (~> 0.4)
rubocop (~> 0.93)
rubocop (~> 1.3)
rubocop-rails (~> 2.8)
ruby-progressbar (~> 1.10)
sanitize (~> 5.2)
@@ -795,12 +794,11 @@ DEPENDENCIES
streamio-ffmpeg (~> 3.0)
strong_migrations (~> 0.7)
thor (~> 1.0)
thwait (~> 0.2.0)
tty-prompt (~> 0.22)
twitter-text (~> 1.14)
tzinfo-data (~> 1.2020)
webauthn (~> 3.0.0.alpha1)
webmock (~> 3.9)
webmock (~> 3.10)
webpacker (~> 5.2)
webpush
xorcist (~> 1.1)

+ 19
- 0
app/controllers/settings/exports/bookmarks_controller.rb View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true

module Settings
module Exports
class BookmarksController < BaseController
include ExportControllerConcern

def index
send_export_file
end

private

def export_data
@export.to_bookmarks_csv
end
end
end
end

+ 7
- 0
app/javascript/mastodon/actions/app.js View File

@@ -8,3 +8,10 @@ export const focusApp = () => ({
export const unfocusApp = () => ({
type: APP_UNFOCUS,
});

export const APP_LAYOUT_CHANGE = 'APP_LAYOUT_CHANGE';

export const changeLayout = layout => ({
type: APP_LAYOUT_CHANGE,
layout,
});

+ 9
- 6
app/javascript/mastodon/components/status.js View File

@@ -97,7 +97,10 @@ class Status extends ImmutablePureComponent {
cachedMediaWidth: PropTypes.number,
scrollKey: PropTypes.string,
deployPictureInPicture: PropTypes.func,
usingPiP: PropTypes.bool,
pictureInPicture: PropTypes.shape({
inUse: PropTypes.bool,
available: PropTypes.bool,
}),
};

// Avoid checking props that are functions (and whose equality will always
@@ -108,7 +111,7 @@ class Status extends ImmutablePureComponent {
'muted',
'hidden',
'unread',
'usingPiP',
'pictureInPicture',
];

state = {
@@ -277,7 +280,7 @@ class Status extends ImmutablePureComponent {
let media = null;
let statusAvatar, prepend, rebloggedByText;

const { intl, hidden, featured, otherAccounts, unread, showThread, scrollKey, usingPiP } = this.props;
const { intl, hidden, featured, otherAccounts, unread, showThread, scrollKey, pictureInPicture } = this.props;

let { status, account, ...other } = this.props;

@@ -348,7 +351,7 @@ class Status extends ImmutablePureComponent {
status = status.get('reblog');
}

if (usingPiP) {
if (pictureInPicture.inUse) {
media = <PictureInPicturePlaceholder width={this.props.cachedMediaWidth} />;
} else if (status.get('media_attachments').size > 0) {
if (this.props.muted) {
@@ -375,7 +378,7 @@ class Status extends ImmutablePureComponent {
width={this.props.cachedMediaWidth}
height={110}
cacheWidth={this.props.cacheMediaWidth}
deployPictureInPicture={this.handleDeployPictureInPicture}
deployPictureInPicture={pictureInPicture.available ? this.handleDeployPictureInPicture : undefined}
/>
)}
</Bundle>
@@ -397,7 +400,7 @@ class Status extends ImmutablePureComponent {
sensitive={status.get('sensitive')}
onOpenVideo={this.handleOpenVideo}
cacheWidth={this.props.cacheMediaWidth}
deployPictureInPicture={this.handleDeployPictureInPicture}
deployPictureInPicture={pictureInPicture.available ? this.handleDeployPictureInPicture : undefined}
visible={this.state.showMedia}
onToggleVisibility={this.handleToggleMediaVisibility}
/>


+ 5
- 1
app/javascript/mastodon/containers/status_container.js View File

@@ -57,7 +57,11 @@ const makeMapStateToProps = () => {

const mapStateToProps = (state, props) => ({
status: getStatus(state, props),
usingPiP: state.get('picture_in_picture').statusId === props.id,

pictureInPicture: {
inUse: state.getIn(['meta', 'layout']) !== 'mobile' && state.get('picture_in_picture').statusId === props.id,
available: state.getIn(['meta', 'layout']) !== 'mobile',
},
});

return mapStateToProps;


+ 50
- 1
app/javascript/mastodon/features/audio/index.js View File

@@ -386,13 +386,59 @@ class Audio extends React.PureComponent {
return this.props.foregroundColor || '#ffffff';
}

seekBy (time) {
const currentTime = this.audio.currentTime + time;

if (!isNaN(currentTime)) {
this.setState({ currentTime }, () => {
this.audio.currentTime = currentTime;
});
}
}

handleAudioKeyDown = e => {
// On the audio element or the seek bar, we can safely use the space bar
// for playback control because there are no buttons to press

if (e.key === ' ') {
e.preventDefault();
e.stopPropagation();
this.togglePlay();
}
}

handleKeyDown = e => {
switch(e.key) {
case 'k':
e.preventDefault();
e.stopPropagation();
this.togglePlay();
break;
case 'm':
e.preventDefault();
e.stopPropagation();
this.toggleMute();
break;
case 'j':
e.preventDefault();
e.stopPropagation();
this.seekBy(-10);
break;
case 'l':
e.preventDefault();
e.stopPropagation();
this.seekBy(10);
break;
}
}

render () {
const { src, intl, alt, editable, autoPlay } = this.props;
const { paused, muted, volume, currentTime, duration, buffer, dragging } = this.state;
const progress = Math.min((currentTime / duration) * 100, 100);

return (
<div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave}>
<div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave} tabIndex='0' onKeyDown={this.handleKeyDown}>
<audio
src={src}
ref={this.setAudioRef}
@@ -406,12 +452,14 @@ class Audio extends React.PureComponent {

<canvas
role='button'
tabIndex='0'
className='audio-player__canvas'
width={this.state.width}
height={this.state.height}
style={{ width: '100%', position: 'absolute', top: 0, left: 0 }}
ref={this.setCanvasRef}
onClick={this.togglePlay}
onKeyDown={this.handleAudioKeyDown}
title={alt}
aria-label={alt}
/>
@@ -432,6 +480,7 @@ class Audio extends React.PureComponent {
className={classNames('video-player__seek__handle', { active: dragging })}
tabIndex='0'
style={{ left: `${progress}%`, backgroundColor: this._getAccentColor() }}
onKeyDown={this.handleAudioKeyDown}
/>
</div>



+ 1
- 1
app/javascript/mastodon/features/getting_started/components/announcements.js View File

@@ -396,7 +396,7 @@ class Announcements extends ImmutablePureComponent {
_markAnnouncementAsRead () {
const { dismissAnnouncement, announcements } = this.props;
const { index } = this.state;
const announcement = announcements.get(index);
const announcement = announcements.get(announcements.size - 1 - index);
if (!announcement.get('read')) dismissAnnouncement(announcement.get('id'));
}



+ 37
- 54
app/javascript/mastodon/features/ui/index.js View File

@@ -8,14 +8,14 @@ import PropTypes from 'prop-types';
import NotificationsContainer from './containers/notifications_container';
import LoadingBarContainer from './containers/loading_bar_container';
import ModalContainer from './containers/modal_container';
import { isMobile } from '../../is_mobile';
import { layoutFromWindow } from 'mastodon/is_mobile';
import { debounce } from 'lodash';
import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose';
import { expandHomeTimeline } from '../../actions/timelines';
import { expandNotifications } from '../../actions/notifications';
import { fetchFilters } from '../../actions/filters';
import { clearHeight } from '../../actions/height_cache';
import { focusApp, unfocusApp } from 'mastodon/actions/app';
import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app';
import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers';
import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
import UploadArea from './components/upload_area';
@@ -52,7 +52,7 @@ import {
Search,
Directory,
} from './util/async-components';
import { me, forceSingleColumn } from '../../initial_state';
import { me } from '../../initial_state';
import { previewState as previewMediaState } from './components/media_modal';
import { previewState as previewVideoState } from './components/video_modal';

@@ -65,6 +65,7 @@ const messages = defineMessages({
});

const mapStateToProps = state => ({
layout: state.getIn(['meta', 'layout']),
isComposing: state.getIn(['compose', 'is_composing']),
hasComposingText: state.getIn(['compose', 'text']).trim().length !== 0,
hasMediaAttachments: state.getIn(['compose', 'media_attachments']).size > 0,
@@ -110,17 +111,11 @@ class SwitchingColumnsArea extends React.PureComponent {
static propTypes = {
children: PropTypes.node,
location: PropTypes.object,
onLayoutChange: PropTypes.func.isRequired,
};

state = {
mobile: isMobile(window.innerWidth),
mobile: PropTypes.bool,
};

componentWillMount () {
window.addEventListener('resize', this.handleResize, { passive: true });

if (this.state.mobile || forceSingleColumn) {
if (this.props.mobile) {
document.body.classList.toggle('layout-single-column', true);
document.body.classList.toggle('layout-multiple-columns', false);
} else {
@@ -129,44 +124,21 @@ class SwitchingColumnsArea extends React.PureComponent {
}
}

componentDidUpdate (prevProps, prevState) {
componentDidUpdate (prevProps) {
if (![this.props.location.pathname, '/'].includes(prevProps.location.pathname)) {
this.node.handleChildrenContentChange();
}

if (prevState.mobile !== this.state.mobile && !forceSingleColumn) {
document.body.classList.toggle('layout-single-column', this.state.mobile);
document.body.classList.toggle('layout-multiple-columns', !this.state.mobile);
if (prevProps.mobile !== this.props.mobile) {
document.body.classList.toggle('layout-single-column', this.props.mobile);
document.body.classList.toggle('layout-multiple-columns', !this.props.mobile);
}
}

componentWillUnmount () {
window.removeEventListener('resize', this.handleResize);
}

shouldUpdateScroll (_, { location }) {
return location.state !== previewMediaState && location.state !== previewVideoState;
}

handleLayoutChange = debounce(() => {
// The cached heights are no longer accurate, invalidate
this.props.onLayoutChange();
}, 500, {
trailing: true,
})

handleResize = () => {
const mobile = isMobile(window.innerWidth);

if (mobile !== this.state.mobile) {
this.handleLayoutChange.cancel();
this.props.onLayoutChange();
this.setState({ mobile });
} else {
this.handleLayoutChange();
}
}

setRef = c => {
if (c) {
this.node = c.getWrappedInstance();
@@ -174,13 +146,11 @@ class SwitchingColumnsArea extends React.PureComponent {
}

render () {
const { children } = this.props;
const { mobile } = this.state;
const singleColumn = forceSingleColumn || mobile;
const redirect = singleColumn ? <Redirect from='/' to='/timelines/home' exact /> : <Redirect from='/' to='/getting-started' exact />;
const { children, mobile } = this.props;
const redirect = mobile ? <Redirect from='/' to='/timelines/home' exact /> : <Redirect from='/' to='/getting-started' exact />;

return (
<ColumnsAreaContainer ref={this.setRef} singleColumn={singleColumn}>
<ColumnsAreaContainer ref={this.setRef} singleColumn={mobile}>
<WrappedSwitch>
{redirect}
<WrappedRoute path='/getting-started' component={GettingStarted} content={children} />
@@ -244,6 +214,7 @@ class UI extends React.PureComponent {
location: PropTypes.object,
intl: PropTypes.object.isRequired,
dropdownMenuIsOpen: PropTypes.bool,
layout: PropTypes.string.isRequired,
};

state = {
@@ -273,11 +244,6 @@ class UI extends React.PureComponent {
this.props.dispatch(unfocusApp());
}

handleLayoutChange = () => {
// The cached heights are no longer accurate, invalidate
this.props.dispatch(clearHeight());
}

handleDragEnter = (e) => {
e.preventDefault();

@@ -351,10 +317,28 @@ class UI extends React.PureComponent {
}
}

componentWillMount () {
handleLayoutChange = debounce(() => {
this.props.dispatch(clearHeight()); // The cached heights are no longer accurate, invalidate
}, 500, {
trailing: true,
});

handleResize = () => {
const layout = layoutFromWindow();

if (layout !== this.props.layout) {
this.handleLayoutChange.cancel();
this.props.dispatch(changeLayout(layout));
} else {
this.handleLayoutChange();
}
}

componentDidMount () {
window.addEventListener('focus', this.handleWindowFocus, false);
window.addEventListener('blur', this.handleWindowBlur, false);
window.addEventListener('beforeunload', this.handleBeforeUnload, false);
window.addEventListener('resize', this.handleResize, { passive: true });

document.addEventListener('dragenter', this.handleDragEnter, false);
document.addEventListener('dragover', this.handleDragOver, false);
@@ -371,9 +355,7 @@ class UI extends React.PureComponent {
this.props.dispatch(expandNotifications());

setTimeout(() => this.props.dispatch(fetchFilters()), 500);
}

componentDidMount () {
this.hotkeys.__mousetrap__.stopCallback = (e, element) => {
return ['TEXTAREA', 'SELECT', 'INPUT'].includes(element.tagName);
};
@@ -383,6 +365,7 @@ class UI extends React.PureComponent {
window.removeEventListener('focus', this.handleWindowFocus);
window.removeEventListener('blur', this.handleWindowBlur);
window.removeEventListener('beforeunload', this.handleBeforeUnload);
window.removeEventListener('resize', this.handleResize);

document.removeEventListener('dragenter', this.handleDragEnter);
document.removeEventListener('dragover', this.handleDragOver);
@@ -513,7 +496,7 @@ class UI extends React.PureComponent {

render () {
const { draggingOver } = this.state;
const { children, isComposing, location, dropdownMenuIsOpen } = this.props;
const { children, isComposing, location, dropdownMenuIsOpen, layout } = this.props;

const handlers = {
help: this.handleHotkeyToggleHelp,
@@ -540,11 +523,11 @@ class UI extends React.PureComponent {
return (
<HotKeys keyMap={keyMap} handlers={handlers} ref={this.setHotkeysRef} attach={window} focused>
<div className={classNames('ui', { 'is-composing': isComposing })} ref={this.setRef} style={{ pointerEvents: dropdownMenuIsOpen ? 'none' : null }}>
<SwitchingColumnsArea location={location} onLayoutChange={this.handleLayoutChange}>
<SwitchingColumnsArea location={location} mobile={layout === 'mobile' || layout === 'single-column'}>
{children}
</SwitchingColumnsArea>

<PictureInPicture />
{layout !== 'mobile' && <PictureInPicture />}
<NotificationsContainer />
<LoadingBarContainer className='loading-bar' />
<ModalContainer />


+ 78
- 0
app/javascript/mastodon/features/video/index.js View File

@@ -266,6 +266,81 @@ class Video extends React.PureComponent {
}
}, 15);

seekBy (time) {
const currentTime = this.video.currentTime + time;

if (!isNaN(currentTime)) {
this.setState({ currentTime }, () => {
this.video.currentTime = currentTime;
});
}
}

handleVideoKeyDown = e => {
// On the video element or the seek bar, we can safely use the space bar
// for playback control because there are no buttons to press

if (e.key === ' ') {
e.preventDefault();
e.stopPropagation();
this.togglePlay();
}
}

handleKeyDown = e => {
const frameTime = 1 / 25;

switch(e.key) {
case 'k':
e.preventDefault();
e.stopPropagation();
this.togglePlay();
break;
case 'm':
e.preventDefault();
e.stopPropagation();
this.toggleMute();
break;
case 'f':
e.preventDefault();
e.stopPropagation();
this.toggleFullscreen();
break;
case 'j':
e.preventDefault();
e.stopPropagation();
this.seekBy(-10);
break;
case 'l':
e.preventDefault();
e.stopPropagation();
this.seekBy(10);
break;
case ',':
e.preventDefault();
e.stopPropagation();
this.seekBy(-frameTime);
break;
case '.':
e.preventDefault();
e.stopPropagation();
this.seekBy(frameTime);
break;
}

// If we are in fullscreen mode, we don't want any hotkeys
// interacting with the UI that's not visible

if (this.state.fullscreen) {
e.preventDefault();
e.stopPropagation();

if (e.key === 'Escape') {
exitFullscreen();
}
}
}

togglePlay = () => {
if (this.state.paused) {
this.setState({ paused: false }, () => this.video.play());
@@ -484,6 +559,7 @@ class Video extends React.PureComponent {
onMouseEnter={this.handleMouseEnter}
onMouseLeave={this.handleMouseLeave}
onClick={this.handleClickRoot}
onKeyDown={this.handleKeyDown}
tabIndex={0}
>
<Blurhash
@@ -507,6 +583,7 @@ class Video extends React.PureComponent {
height={height}
volume={volume}
onClick={this.togglePlay}
onKeyDown={this.handleVideoKeyDown}
onPlay={this.handlePlay}
onPause={this.handlePause}
onLoadedData={this.handleLoadedData}
@@ -529,6 +606,7 @@ class Video extends React.PureComponent {
className={classNames('video-player__seek__handle', { active: dragging })}
tabIndex='0'
style={{ left: `${progress}%` }}
onKeyDown={this.handleVideoKeyDown}
/>
</div>



+ 15
- 10
app/javascript/mastodon/is_mobile.js View File

@@ -1,9 +1,18 @@
import { supportsPassiveEvents } from 'detect-passive-events';
import { forceSingleColumn } from 'mastodon/initial_state';

const LAYOUT_BREAKPOINT = 630;

export function isMobile(width) {
return width <= LAYOUT_BREAKPOINT;
export const isMobile = width => width <= LAYOUT_BREAKPOINT;

export const layoutFromWindow = () => {
if (isMobile(window.innerWidth)) {
return 'mobile';
} else if (forceSingleColumn) {
return 'single-column';
} else {
return 'multi-column';
}
};

const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream;
@@ -11,17 +20,13 @@ const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream;
let userTouching = false;
let listenerOptions = supportsPassiveEvents ? { passive: true } : false;

function touchListener() {
const touchListener = () => {
userTouching = true;
window.removeEventListener('touchstart', touchListener, listenerOptions);
}
};

window.addEventListener('touchstart', touchListener, listenerOptions);

export function isUserTouching() {
return userTouching;
}
export const isUserTouching = () => userTouching;

export function isIOS() {
return iOS;
};
export const isIOS = () => iOS;

+ 6
- 1
app/javascript/mastodon/reducers/meta.js View File

@@ -1,15 +1,20 @@
import { STORE_HYDRATE } from '../actions/store';
import { STORE_HYDRATE } from 'mastodon/actions/store';
import { APP_LAYOUT_CHANGE } from 'mastodon/actions/app';
import { Map as ImmutableMap } from 'immutable';
import { layoutFromWindow } from 'mastodon/is_mobile';

const initialState = ImmutableMap({
streaming_api_base_url: null,
access_token: null,
layout: layoutFromWindow(),
});

export default function meta(state = initialState, action) {
switch(action.type) {
case STORE_HYDRATE:
return state.merge(action.state.get('meta'));
case APP_LAYOUT_CHANGE:
return state.set('layout', action.layout);
default:
return state;
}


+ 41
- 9
app/javascript/mastodon/utils/resize_image.js View File

@@ -41,6 +41,45 @@ const dropOrientationIfNeeded = (orientation) => new Promise(resolve => {
}
});

// Some browsers don't allow reading from a canvas and instead return all-white
// or randomized data. Use a pre-defined image to check if reading the canvas
// works.
const checkCanvasReliability = () => new Promise((resolve, reject) => {
switch(_browser_quirks['canvas-read-unreliable']) {
case true:
reject('Canvas reading unreliable');
break;
case false:
resolve();
break;
default:
// 2×2 GIF with white, red, green and blue pixels
const testImageURL =
'data:image/gif;base64,R0lGODdhAgACAKEDAAAA//8AAAD/AP///ywAAAAAAgACAAACA1wEBQA7';
const refData =
[255, 255, 255, 255, 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255];
const img = new Image();
img.onload = () => {
const canvas = document.createElement('canvas');
const context = canvas.getContext('2d');
context.drawImage(img, 0, 0, 2, 2);
const imageData = context.getImageData(0, 0, 2, 2);
if (imageData.data.every((x, i) => refData[i] === x)) {
_browser_quirks['canvas-read-unreliable'] = false;
resolve();
} else {
_browser_quirks['canvas-read-unreliable'] = true;
reject('Canvas reading unreliable');
}
};
img.onerror = () => {
_browser_quirks['canvas-read-unreliable'] = true;
reject('Failed to load test image');
};
img.src = testImageURL;
}
});

const getImageUrl = inputFile => new Promise((resolve, reject) => {
if (window.URL && URL.createObjectURL) {
try {
@@ -110,14 +149,6 @@ const processImage = (img, { width, height, orientation, type = 'image/png' }) =

context.drawImage(img, 0, 0, width, height);

// The Tor Browser and maybe other browsers may prevent reading from canvas
// and return an all-white image instead. Assume reading failed if the resized
// image is perfectly white.
const imageData = context.getImageData(0, 0, width, height);
if (imageData.data.every(value => value === 255)) {
throw 'Failed to read from canvas';
}

canvas.toBlob(resolve, type);
});

@@ -127,7 +158,8 @@ const resizeImage = (img, type = 'image/png') => new Promise((resolve, reject) =
const newWidth = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (width / height)));
const newHeight = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (height / width)));

getOrientation(img, type)
checkCanvasReliability()
.then(getOrientation(img, type))
.then(orientation => processImage(img, {
width: newWidth,
height: newHeight,


+ 1
- 1
app/lib/activitypub/activity/delete.rb View File

@@ -13,7 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity

def delete_person
lock_or_return("delete_in_progress:#{@account.id}") do
DeleteAccountService.new.call(@account, reserve_username: false)
DeleteAccountService.new.call(@account, reserve_username: false, skip_activitypub: true)
end
end



+ 28
- 0
app/lib/cache_buster.rb View File

@@ -0,0 +1,28 @@
# frozen_string_literal: true

class CacheBuster
def initialize(options = {})
@secret_header = options[:secret_header] || 'Secret-Header'
@secret = options[:secret] || 'True'
end

def bust(url)
site = Addressable::URI.parse(url).normalized_site

request_pool.with(site) do |http_client|
build_request(url, http_client).perform
end
end

private

def request_pool
RequestPool.current
end

def build_request(url, http_client)
Request.new(:get, url, http_client: http_client).tap do |request|
request.add_headers(@secret_header => @secret)
end
end
end

+ 12
- 0
app/models/export.rb View File

@@ -9,6 +9,14 @@ class Export
@account = account
end

def to_bookmarks_csv
CSV.generate do |csv|
account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark|
csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)]
end
end
end

def to_blocked_accounts_csv
to_csv account.blocking.select(:username, :domain)
end
@@ -55,6 +63,10 @@ class Export
account.statuses_count
end

def total_bookmarks
account.bookmarks.count
end

def total_follows
account.following_count
end


+ 1
- 1
app/models/import.rb View File

@@ -24,7 +24,7 @@ class Import < ApplicationRecord

belongs_to :account

enum type: [:following, :blocking, :muting, :domain_blocking]
enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks]

validates :type, presence: true



+ 4
- 1
app/services/delete_account_service.rb View File

@@ -41,6 +41,7 @@ class DeleteAccountService < BaseService
# @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts
# @option [Boolean] :reserve_username Keep account record
# @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads
# @option [Boolean] :skip_activitypub Skip sending ActivityPub payloads. Implied by :skip_side_effects
# @option [Time] :suspended_at Only applicable when :reserve_username is true
def call(account, **options)
@account = account
@@ -52,6 +53,8 @@ class DeleteAccountService < BaseService
@options[:skip_side_effects] = true
end

@options[:skip_activitypub] = true if @options[:skip_side_effects]

reject_follows!
purge_user!
purge_profile!
@@ -62,7 +65,7 @@ class DeleteAccountService < BaseService
private

def reject_follows!
return if @account.local? || !@account.activitypub?
return if @account.local? || !@account.activitypub? || @options[:skip_activitypub]

# When deleting a remote account, the account obviously doesn't
# actually become deleted on its origin server, i.e. unlike a


+ 45
- 0
app/services/import_service.rb View File

@@ -18,6 +18,8 @@ class ImportService < BaseService
import_mutes!
when 'domain_blocking'
import_domain_blocks!
when 'bookmarks'
import_bookmarks!
end
end

@@ -88,6 +90,39 @@ class ImportService < BaseService
end
end

def import_bookmarks!
parse_import_data!(['#uri'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }

if @import.overwrite?
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }

@account.bookmarks.find_each do |bookmark|
if presence_hash[bookmark.status.uri]
items.delete(bookmark.status.uri)
else
bookmark.destroy!
end
end
end

statuses = items.map do |uri|
status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)

status || ActivityPub::FetchRemoteStatusService.new.call(uri)
end.compact

account_ids = statuses.map(&:account_id)
preloaded_relations = relations_map_for_account(@account, account_ids)

statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }

statuses.each do |status|
@account.bookmarks.find_or_create_by!(account: @account, status: status)
end
end

def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@@ -101,4 +136,14 @@ class ImportService < BaseService
def follow_limit
FollowLimitValidator.limit_for_account(@account)
end

def relations_map_for_account(account, account_ids)
{
blocking: {},
blocked_by: Account.blocked_by_map(account_ids, account.id),
muting: {},
following: Account.following_map(account_ids, account.id),
domain_blocking_by_domain: {},
}
end
end

+ 16
- 9
app/services/resolve_account_service.rb View File

@@ -29,6 +29,7 @@ class ResolveAccountService < BaseService
# At this point we are in need of a Webfinger query, which may
# yield us a different username/domain through a redirect
process_webfinger!(@uri)
@domain = nil if TagManager.instance.local_domain?(@domain)

# Because the username/domain pair may be different than what
# we already checked, we need to check if we've already got
@@ -78,25 +79,31 @@ class ResolveAccountService < BaseService
@uri = [@username, @domain].compact.join('@')
end

def process_webfinger!(uri, redirected = false)
def process_webfinger!(uri)
@webfinger = webfinger!("acct:#{uri}")
confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
confirmed_username, confirmed_domain = split_acct(@webfinger.subject)

if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
@username = confirmed_username
@domain = confirmed_domain
@uri = uri
elsif !redirected
return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true)
else
raise Webfinger::RedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
return
end

@domain = nil if TagManager.instance.local_domain?(@domain)
# Account doesn't match, so it may have been redirected
@webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}")
@username, @domain = split_acct(@webfinger.subject)

unless confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
raise Webfinger::RedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
end
rescue Webfinger::GoneError
@gone = true
end

def split_acct(acct)
acct.gsub(/\Aacct:/, '').split('@')
end

def process_account!
return unless activitypub_ready?

@@ -145,7 +152,7 @@ class ResolveAccountService < BaseService
end

def queue_deletion!
AccountDeletionWorker.perform_async(@account.id, reserve_username: false)
AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
end

def lock_options


+ 2
- 0
app/services/suspend_account_service.rb View File

@@ -78,6 +78,8 @@ class SuspendAccountService < BaseService
Rails.logger.warn "Tried to change permission on non-existent file #{attachment.path(style)}"
end
end

CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
end
end
end


+ 2
- 0
app/services/unsuspend_account_service.rb View File

@@ -69,6 +69,8 @@ class UnsuspendAccountService < BaseService
Rails.logger.warn "Tried to change permission on non-existent file #{attachment.path(style)}"
end
end

CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
end
end
end


+ 4
- 0
app/views/settings/exports/show.html.haml View File

@@ -36,6 +36,10 @@
%th= t('exports.domain_blocks')
%td= number_with_delimiter @export.total_domain_blocks
%td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
%tr
%th= t('exports.bookmarks')
%td= number_with_delimiter @export.total_bookmarks
%td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv)

%hr.spacer/



+ 2
- 1
app/workers/account_deletion_worker.rb View File

@@ -7,7 +7,8 @@ class AccountDeletionWorker

def perform(account_id, options = {})
reserve_username = options.with_indifferent_access.fetch(:reserve_username, true)
DeleteAccountService.new.call(Account.find(account_id), reserve_username: reserve_username, reserve_email: false)
skip_activitypub = options.with_indifferent_access.fetch(:skip_activitypub, false)
DeleteAccountService.new.call(Account.find(account_id), reserve_username: reserve_username, skip_activitypub: skip_activitypub, reserve_email: false)
rescue ActiveRecord::RecordNotFound
true
end


+ 18
- 0
app/workers/cache_buster_worker.rb View File

@@ -0,0 +1,18 @@
# frozen_string_literal: true

class CacheBusterWorker
include Sidekiq::Worker
include RoutingHelper

sidekiq_options queue: 'pull'

def perform(path)
cache_buster.bust(full_asset_url(path))
end

private

def cache_buster
CacheBuster.new(Rails.configuration.x.cache_buster)
end
end

+ 10
- 0
config/initializers/cache_buster.rb View File

@@ -0,0 +1,10 @@
# frozen_string_literal: true

Rails.application.configure do
config.x.cache_buster_enabled = ENV['CACHE_BUSTER_ENABLED'] == 'true'

config.x.cache_buster = {
secret_header: ENV['CACHE_BUSTER_SECRET_HEADER'],
secret: ENV['CACHE_BUSTER_SECRET'],
}
end

+ 0
- 1
config/initializers/paperclip.rb View File

@@ -107,7 +107,6 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
else
Paperclip::Attachment.default_options.merge!(
storage: :filesystem,
use_timestamp: true,
path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':prefix_path:class', ':attachment', ':id_partition', ':style', ':filename'),
url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:prefix_url:class/:attachment/:id_partition/:style/:filename',
)


+ 2
- 0
config/locales/en.yml View File

@@ -842,6 +842,7 @@ en:
request: Request your archive
size: Size
blocks: You block
bookmarks: Bookmarks
csv: CSV
domain_blocks: Domain blocks
lists: Lists
@@ -918,6 +919,7 @@ en:
success: Your data was successfully uploaded and will now be processed in due time
types:
blocking: Blocking list
bookmarks: Bookmarks
domain_blocking: Domain blocking list
following: Following list
muting: Muting list


+ 1
- 0
config/routes.rb View File

@@ -125,6 +125,7 @@ Rails.application.routes.draw do
resources :mutes, only: :index, controller: :muted_accounts
resources :lists, only: :index, controller: :lists
resources :domain_blocks, only: :index, controller: :blocked_domains
resources :bookmarks, only: :index, controller: :bookmarks
end

resources :two_factor_authentication_methods, only: [:index] do


+ 4
- 0
lib/cli.rb View File

@@ -14,6 +14,7 @@ require_relative 'mastodon/cache_cli'
require_relative 'mastodon/upgrade_cli'
require_relative 'mastodon/email_domain_blocks_cli'
require_relative 'mastodon/ip_blocks_cli'
require_relative 'mastodon/maintenance_cli'
require_relative 'mastodon/version'

module Mastodon
@@ -61,6 +62,9 @@ module Mastodon
desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks'
subcommand 'ip_blocks', Mastodon::IpBlocksCLI

desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
subcommand 'maintenance', Mastodon::MaintenanceCLI

option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC


+ 610
- 0
lib/mastodon/maintenance_cli.rb View File

@@ -0,0 +1,610 @@
# frozen_string_literal: true

require 'tty-prompt'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'

module Mastodon
class MaintenanceCLI < Thor
include CLIHelper

def self.exit_on_failure?
true
end

MIN_SUPPORTED_VERSION = 2019_10_01_213028
MAX_SUPPORTED_VERSION = 2020_10_17_234926

# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database

class Status < ApplicationRecord; end
class StatusPin < ApplicationRecord; end
class Poll < ApplicationRecord; end
class Report < ApplicationRecord; end
class Tombstone < ApplicationRecord; end
class Favourite < ApplicationRecord; end
class Follow < ApplicationRecord; end
class FollowRequest < ApplicationRecord; end
class Block < ApplicationRecord; end
class Mute < ApplicationRecord; end
class AccountIdentityProof < ApplicationRecord; end
class AccountModerationNote < ApplicationRecord; end
class AccountPin < ApplicationRecord; end
class ListAccount < ApplicationRecord; end
class PollVote < ApplicationRecord; end
class Mention < ApplicationRecord; end
class AccountDomainBlock < ApplicationRecord; end
class AnnouncementReaction < ApplicationRecord; end
class FeaturedTag < ApplicationRecord; end
class CustomEmoji < ApplicationRecord; end
class CustomEmojiCategory < ApplicationRecord; end
class Bookmark < ApplicationRecord; end
class WebauthnCredential < ApplicationRecord; end

class PreviewCard < ApplicationRecord
self.inheritance_column = false
end

class MediaAttachment < ApplicationRecord
self.inheritance_column = nil
end

class AccountStat < ApplicationRecord
belongs_to :account, inverse_of: :account_stat
end

class Account < ApplicationRecord
# Dummy class, to make migration possible across version changes
has_one :user, inverse_of: :account
has_one :account_stat, inverse_of: :account

scope :local, -> { where(domain: nil) }

def local?
domain.nil?
end

def acct
local? ? username : "#{username}@#{domain}"
end
end

class User < ApplicationRecord
belongs_to :account, inverse_of: :user
end

desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes'
long_desc <<~LONG_DESC
Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes.

This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes

Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
LONG_DESC
def fix_duplicates
@prompt = TTY::Prompt.new

if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
@prompt.warn 'Your version of the database schema is too old and is not supported by this script.'
@prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.'
exit(1)
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
@prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.'
exit(1) unless @prompt.yes?('Continue anyway?')
end

@prompt.warn 'This task will take a long time to run and is potentially destructive.'
@prompt.warn 'Please make sure to stop Mastodon and have a backup.'
exit(1) unless @prompt.yes?('Continue?')

deduplicate_accounts!
deduplicate_users!
deduplicate_account_domain_blocks!
deduplicate_account_identity_proofs!
deduplicate_announcement_reactions!
deduplicate_conversations!
deduplicate_custom_emojis!
deduplicate_custom_emoji_categories!
deduplicate_domain_allows!
deduplicate_domain_blocks!
deduplicate_unavailable_domains!
deduplicate_email_domain_blocks!
deduplicate_media_attachments!
deduplicate_preview_cards!
deduplicate_statuses!
deduplicate_tags!
deduplicate_webauthn_credentials!

Rails.cache.clear

@prompt.say 'Finished!'
end

private

def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')

@prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'

find_duplicate_accounts.each do |row|
accounts = Account.where(id: row['ids'].split(',')).to_a

if accounts.first.local?
deduplicate_local_accounts!(accounts)
else
deduplicate_remote_accounts!(accounts)
end
end

@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
if ActiveRecord::Migrator.current_version < 20200620164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
end
end

def deduplicate_users!
remove_index_if_exists!(:users, 'index_users_on_confirmation_token')
remove_index_if_exists!(:users, 'index_users_on_email')
remove_index_if_exists!(:users, 'index_users_on_remember_token')
remove_index_if_exists!(:users, 'index_users_on_reset_password_token')

@prompt.say 'Deduplicating user records…'

# Deduplicating email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
@prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
@prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
@prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'

i = 0
users.each do |user|
user.update!(email: "#{i} " + user.email)
end
end

ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"

users.each do |user|
user.update!(confirmation_token: nil)
end
end

ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"

users.each do |user|
user.update!(remember_token: nil)
end
end

ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"

users.each do |user|
user.update!(reset_password_token: nil)
end
end

@prompt.say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
end

def deduplicate_account_domain_blocks!
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')

@prompt.say 'Removing duplicate account domain blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
end

@prompt.say 'Restoring account domain blocks indexes…'
ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
end

def deduplicate_account_identity_proofs!
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')

@prompt.say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring account identity proofs indexes…'
ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
end

def deduplicate_announcement_reactions!
return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions)

remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')

@prompt.say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring announcement_reactions indexes…'
ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
end

def deduplicate_conversations!
remove_index_if_exists!(:conversations, 'index_conversations_on_uri')

@prompt.say 'Deduplicating conversations…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse

ref_conversation = conversations.shift

conversations.each do |other|
merge_conversations!(ref_conversation, other)
other.destroy
end
end

@prompt.say 'Restoring conversations indexes…'
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
end

def deduplicate_custom_emojis!
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')

@prompt.say 'Deduplicating custom_emojis…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse

ref_emoji = emojis.shift

emojis.each do |other|
merge_custom_emojis!(ref_emoji, other)
other.destroy
end
end

@prompt.say 'Restoring custom_emojis indexes…'
ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
end

def deduplicate_custom_emoji_categories!
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')

@prompt.say 'Deduplicating custom_emoji_categories…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse

ref_category = categories.shift

categories.each do |other|
merge_custom_emoji_categories!(ref_category, other)
other.destroy
end
end

@prompt.say 'Restoring custom_emoji_categories indexes…'
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
end

def deduplicate_domain_allows!
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')

@prompt.say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
end

def deduplicate_domain_blocks!
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')

@prompt.say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a

reject_media = domain_blocks.any?(&:reject_media?)
reject_reports = domain_blocks.any?(&:reject_reports?)

reference_block = domain_blocks.shift

private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence }
public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence }

reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment)

domain_blocks.each(&:destroy)
end

@prompt.say 'Restoring domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
end

def deduplicate_unavailable_domains!
return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains)

remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')

@prompt.say 'Deduplicating unavailable_domains…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
end

def deduplicate_email_domain_blocks!
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')

@prompt.say 'Deduplicating email_domain_blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
domain_blocks.drop(1).each(&:destroy)
end

@prompt.say 'Restoring email_domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
end

def deduplicate_media_attachments!
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')

@prompt.say 'Deduplicating media_attachments…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
end

@prompt.say 'Restoring media_attachments indexes…'
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
end

def deduplicate_preview_cards!
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')

@prompt.say 'Deduplicating preview_cards…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring preview_cards indexes…'
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
end

def deduplicate_statuses!
remove_index_if_exists!(:statuses, 'index_statuses_on_uri')

@prompt.say 'Deduplicating statuses…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
ref_status = statuses.shift
statuses.each do |status|
merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id
status.destroy
end
end

@prompt.say 'Restoring statuses indexes…'
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
end

def deduplicate_tags!
remove_index_if_exists!(:tags, 'index_tags_on_name_lower')

@prompt.say 'Deduplicating tags…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
ref_tag = tags.shift
tags.each do |tag|
merge_tags!(ref_tag, tag)
tag.destroy
end
end

@prompt.say 'Restoring tags indexes…'
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
end

def deduplicate_webauthn_credentials!
return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials)

remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')

@prompt.say 'Deduplicating webauthn_credentials…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end

@prompt.say 'Restoring webauthn_credentials indexes…'
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
end

def deduplicate_local_accounts!(accounts)
accounts = accounts.sort_by(&:id).reverse

@prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
@prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.'

accounts.each_with_index do |account, idx|
@prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A']
end

@prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'

ref_id = @prompt.ask('Account to keep unchanged:') do |q|
q.required true
q.default 0
q.convert :int
end

accounts.delete_at(ref_id)

i = 0
accounts.each do |account|
i += 1
username = account.username + "_#{i}"

while Account.local.exists?(username: username)
i += 1
username = account.username + "_#{i}"
end

account.update!(username: username)
end
end

def deduplicate_remote_accounts!(accounts)
accounts = accounts.sort_by(&:updated_at).reverse

reference_account = accounts.shift

accounts.each do |other_account|
if other_account.public_key == reference_account.public_key
# The accounts definitely point to the same resource, so
# it's safe to re-attribute content and relationships
merge_accounts!(reference_account, other_account)
end

other_account.destroy
end
end

def merge_accounts!(main_account, duplicate_account)
# Since it's the same remote resource, the remote resource likely
# already believes we are following/blocking, so it's safe to
# re-attribute the relationships too. However, during the presence
# of the index bug users could have *also* followed the reference
# account already, therefore mass update will not work and we need
# to check for (and skip past) uniqueness errors
owned_classes = [
Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
Follow, FollowRequest, Block, Mute, AccountIdentityProof,
AccountModerationNote, AccountPin, AccountStat, ListAccount,
PollVote, Mention
]
owned_classes.each do |klass|
klass.where(account_id: duplicate_account.id).find_each do |record|
begin
record.update_attribute(:account_id, main_account.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end

target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
target_classes.each do |klass|
klass.where(target_account_id: duplicate_account.id).find_each do |record|
begin
record.update_attribute(:target_account_id, main_account.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
end

def merge_conversations!(main_conv, duplicate_conv)
owned_classes = [ConversationMute, AccountConversation]
owned_classes.each do |klass|
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
begin
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
end

def merge_custom_emojis!(main_emoji, duplicate_emoji)
owned_classes = [AnnouncementReaction]
owned_classes.each do |klass|
klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id)
end
end

def merge_custom_emoji_categories!(main_category, duplicate_category)
owned_classes = [CustomEmoji]
owned_classes.each do |klass|
klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id)
end
end

def merge_statuses!(main_status, duplicate_status)
owned_classes = [Favourite, Mention, Poll]
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
owned_classes.each do |klass|
klass.where(status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end

StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end

Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end

Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end

def merge_tags!(main_tag, duplicate_tag)
[FeaturedTag].each do |klass|
klass.where(tag_id: duplicate_tag.id).find_each do |record|
begin
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
end

def find_duplicate_accounts
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
end

def remove_index_if_exists!(table, name)
ActiveRecord::Base.connection.remove_index(table, name: name)
rescue ArgumentError
nil
rescue ActiveRecord::StatementInvalid
nil
end
end
end

+ 11
- 0
lib/tasks/db.rake View File

@@ -48,6 +48,17 @@ namespace :db do
end
end

task :post_migration_hook do
at_exit do
unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate'])
Rails.logger.warn 'WARNING: Your database is using an unsafe collation setting, which might result in index corruption.'
Rails.logger.warn 'WARNING: See https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#am-i-affected'
end
end
end

Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])

# Before we load the schema, define the timestamp_id function.
# Idiomatically, we might do this in a migration, but then it
# wouldn't end up in schema.rb, so we'd need to figure out a way to


+ 9
- 9
package.json View File

@@ -77,7 +77,7 @@
"arrow-key-navigation": "^1.2.0",
"autoprefixer": "^9.8.6",
"axios": "^0.21.0",
"babel-loader": "^8.1.0",
"babel-loader": "^8.2.1",
"babel-plugin-lodash": "^3.3.4",
"babel-plugin-preval": "^5.0.0",
"babel-plugin-react-intl": "^6.2.0",
@@ -85,7 +85,7 @@
"babel-runtime": "^6.26.0",
"blurhash": "^1.1.3",
"classnames": "^2.2.5",
"compression-webpack-plugin": "^6.1.0",
"compression-webpack-plugin": "^6.1.1",
"cross-env": "^7.0.2",
"css-loader": "^5.0.1",
"cssnano": "^4.1.10",
@@ -113,7 +113,7 @@
"lodash": "^4.17.19",
"mark-loader": "^0.1.6",
"marky": "^1.2.1",
"mini-css-extract-plugin": "^1.3.0",
"mini-css-extract-plugin": "^1.3.1",
"mkdirp": "^1.0.4",
"npmlog": "^4.1.2",
"object-assign": "^4.1.1",
@@ -156,7 +156,7 @@
"reselect": "^4.0.0",
"rimraf": "^3.0.2",
"sass": "^1.29.0",
"sass-loader": "^10.0.5",
"sass-loader": "^10.1.0",
"stacktrace-js": "^2.0.2",
"stringz": "^2.1.0",
"substring-trie": "^1.0.2",
@@ -168,13 +168,13 @@
"webpack": "^4.44.2",
"webpack-assets-manifest": "^3.1.1",
</