Unverified Commit cdb10134 authored by Eugen Rochko's avatar Eugen Rochko Committed by GitHub
Browse files

Keyword/phrase filtering (#7905)

* Add keyword filtering

    GET|POST       /api/v1/filters
    GET|PUT|DELETE /api/v1/filters/:id

- Irreversible filters can drop toots from home or notifications
- Other filters can hide toots through the client app
- Filters use a phrase valid in particular contexts, expiration

* Make sure expired filters don't get applied client-side

* Add missing API methods

* Remove "regex filter" from column settings

* Add tests

* Add test for FeedManager

* Add CustomFilter test

* Add UI for managing filters

* Add streaming API event to allow syncing filters

* Fix tests
parent fbee9b5a
# frozen_string_literal: true
class Api::V1::FiltersController < Api::BaseController
before_action -> { doorkeeper_authorize! :read }, only: [:index, :show]
before_action -> { doorkeeper_authorize! :write }, except: [:index, :show]
before_action :require_user!
before_action :set_filters, only: :index
before_action :set_filter, only: [:show, :update, :destroy]
respond_to :json
def index
render json: @filters, each_serializer: REST::FilterSerializer
end
def create
@filter = current_account.custom_filters.create!(resource_params)
render json: @filter, serializer: REST::FilterSerializer
end
def show
render json: @filter, serializer: REST::FilterSerializer
end
def update
@filter.update!(resource_params)
render json: @filter, serializer: REST::FilterSerializer
end
def destroy
@filter.destroy!
render_empty
end
private
def set_filters
@filters = current_account.custom_filters
end
def set_filter
@filter = current_account.custom_filters.find(params[:id])
end
def resource_params
params.permit(:phrase, :expires_at, :irreversible, context: [])
end
end
# frozen_string_literal: true
class FiltersController < ApplicationController
include Authorization
layout 'admin'
before_action :set_filters, only: :index
before_action :set_filter, only: [:edit, :update, :destroy]
def index
@filters = current_account.custom_filters
end
def new
@filter = current_account.custom_filters.build
end
def create
@filter = current_account.custom_filters.build(resource_params)
if @filter.save
redirect_to filters_path
else
render action: :new
end
end
def edit; end
def update
if @filter.update(resource_params)
redirect_to filters_path
else
render action: :edit
end
end
def destroy
@filter.destroy
redirect_to filters_path
end
private
def set_filters
@filters = current_account.custom_filters
end
def set_filter
@filter = current_account.custom_filters.find(params[:id])
end
def resource_params
params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, context: [])
end
end
import api from '../api';
export const FILTERS_FETCH_REQUEST = 'FILTERS_FETCH_REQUEST';
export const FILTERS_FETCH_SUCCESS = 'FILTERS_FETCH_SUCCESS';
export const FILTERS_FETCH_FAIL = 'FILTERS_FETCH_FAIL';
export const fetchFilters = () => (dispatch, getState) => {
dispatch({
type: FILTERS_FETCH_REQUEST,
skipLoading: true,
});
api(getState)
.get('/api/v1/filters')
.then(({ data }) => dispatch({
type: FILTERS_FETCH_SUCCESS,
filters: data,
skipLoading: true,
}))
.catch(err => dispatch({
type: FILTERS_FETCH_FAIL,
err,
skipLoading: true,
skipAlert: true,
}));
};
......@@ -6,6 +6,7 @@ import {
disconnectTimeline,
} from './timelines';
import { updateNotifications, expandNotifications } from './notifications';
import { fetchFilters } from './filters';
import { getLocale } from '../locales';
const { messages } = getLocale();
......@@ -30,6 +31,9 @@ export function connectTimelineStream (timelineId, path, pollingRefresh = null)
case 'notification':
dispatch(updateNotifications(JSON.parse(data.payload), messages, locale));
break;
case 'filters_changed':
dispatch(fetchFilters());
break;
}
},
};
......
......@@ -157,6 +157,21 @@ export default class Status extends ImmutablePureComponent {
);
}
if (status.get('filtered') || status.getIn(['reblog', 'filtered'])) {
const minHandlers = this.props.muted ? {} : {
moveUp: this.handleHotkeyMoveUp,
moveDown: this.handleHotkeyMoveDown,
};
return (
<HotKeys handlers={minHandlers}>
<div className='status__wrapper status__wrapper--filtered focusable' tabIndex='0'>
<FormattedMessage id='status.filtered' defaultMessage='Filtered' />
</div>
</HotKeys>
);
}
if (featured) {
prepend = (
<div className='status__prepend'>
......
......@@ -25,6 +25,7 @@ export default class StatusList extends ImmutablePureComponent {
prepend: PropTypes.node,
emptyMessage: PropTypes.node,
alwaysPrepend: PropTypes.bool,
timelineId: PropTypes.string.isRequired,
};
static defaultProps = {
......@@ -70,7 +71,7 @@ export default class StatusList extends ImmutablePureComponent {
}
render () {
const { statusIds, featuredStatusIds, onLoadMore, ...other } = this.props;
const { statusIds, featuredStatusIds, onLoadMore, timelineId, ...other } = this.props;
const { isLoading, isPartial } = other;
if (isPartial) {
......@@ -102,6 +103,7 @@ export default class StatusList extends ImmutablePureComponent {
id={statusId}
onMoveUp={this.handleMoveUp}
onMoveDown={this.handleMoveDown}
contextType={timelineId}
/>
))
) : null;
......@@ -114,6 +116,7 @@ export default class StatusList extends ImmutablePureComponent {
featured
onMoveUp={this.handleMoveUp}
onMoveDown={this.handleMoveDown}
contextType={timelineId}
/>
)).concat(scrollableContent);
}
......
......@@ -42,7 +42,7 @@ const makeMapStateToProps = () => {
const getStatus = makeGetStatus();
const mapStateToProps = (state, props) => ({
status: getStatus(state, props.id),
status: getStatus(state, props),
});
return mapStateToProps;
......
import React from 'react';
import PropTypes from 'prop-types';
import ImmutablePropTypes from 'react-immutable-proptypes';
import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
import SettingText from '../../../components/setting_text';
import { injectIntl, FormattedMessage } from 'react-intl';
import SettingToggle from '../../notifications/components/setting_toggle';
const messages = defineMessages({
filter_regex: { id: 'home.column_settings.filter_regex', defaultMessage: 'Filter out by regular expressions' },
settings: { id: 'home.settings', defaultMessage: 'Column settings' },
});
@injectIntl
export default class ColumnSettings extends React.PureComponent {
......@@ -21,19 +15,13 @@ export default class ColumnSettings extends React.PureComponent {
};
render () {
const { settings, onChange, intl } = this.props;
const { settings, onChange } = this.props;
return (
<div>
<div className='column-settings__row'>
<SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media Only' />} />
</div>
<span className='column-settings__section'><FormattedMessage id='home.column_settings.advanced' defaultMessage='Advanced' /></span>
<div className='column-settings__row'>
<SettingText settings={settings} settingKey={['regex', 'body']} onChange={onChange} label={intl.formatMessage(messages.filter_regex)} />
</div>
</div>
);
}
......
......@@ -7,7 +7,7 @@ const makeMapStateToProps = () => {
const getStatus = makeGetStatus();
const mapStateToProps = state => ({
status: getStatus(state, state.getIn(['compose', 'in_reply_to'])),
status: getStatus(state, { id: state.getIn(['compose', 'in_reply_to']) }),
});
return mapStateToProps;
......
......@@ -7,7 +7,6 @@ import ColumnHeader from '../../components/column_header';
import { expandDirectTimeline } from '../../actions/timelines';
import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
import ColumnSettingsContainer from './containers/column_settings_container';
import { connectDirectStream } from '../../actions/streaming';
const messages = defineMessages({
......@@ -86,9 +85,7 @@ export default class DirectTimeline extends React.PureComponent {
onClick={this.handleHeaderClick}
pinned={pinned}
multiColumn={multiColumn}
>
<ColumnSettingsContainer />
</ColumnHeader>
/>
<StatusListContainer
trackScroll={!pinned}
......
import React from 'react';
import PropTypes from 'prop-types';
import ImmutablePropTypes from 'react-immutable-proptypes';
import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
import { injectIntl, FormattedMessage } from 'react-intl';
import SettingToggle from '../../notifications/components/setting_toggle';
import SettingText from '../../../components/setting_text';
const messages = defineMessages({
filter_regex: { id: 'home.column_settings.filter_regex', defaultMessage: 'Filter out by regular expressions' },
settings: { id: 'home.settings', defaultMessage: 'Column settings' },
});
@injectIntl
export default class ColumnSettings extends React.PureComponent {
......@@ -20,7 +14,7 @@ export default class ColumnSettings extends React.PureComponent {
};
render () {
const { settings, onChange, intl } = this.props;
const { settings, onChange } = this.props;
return (
<div>
......@@ -33,12 +27,6 @@ export default class ColumnSettings extends React.PureComponent {
<div className='column-settings__row'>
<SettingToggle prefix='home_timeline' settings={settings} settingPath={['shows', 'reply']} onChange={onChange} label={<FormattedMessage id='home.column_settings.show_replies' defaultMessage='Show replies' />} />
</div>
<span className='column-settings__section'><FormattedMessage id='home.column_settings.advanced' defaultMessage='Advanced' /></span>
<div className='column-settings__row'>
<SettingText prefix='home_timeline' settings={settings} settingKey={['regex', 'body']} onChange={onChange} label={intl.formatMessage(messages.filter_regex)} />
</div>
</div>
);
}
......
......@@ -58,7 +58,7 @@ const makeMapStateToProps = () => {
const getStatus = makeGetStatus();
const mapStateToProps = (state, props) => {
const status = getStatus(state, props.params.statusId);
const status = getStatus(state, { id: props.params.statusId });
let ancestorsIds = Immutable.List();
let descendantsIds = Immutable.List();
......@@ -336,6 +336,7 @@ export default class Status extends ImmutablePureComponent {
id={id}
onMoveUp={this.handleMoveUp}
onMoveDown={this.handleMoveDown}
contextType='thread'
/>
));
}
......
......@@ -11,15 +11,6 @@ const makeGetStatusIds = () => createSelector([
(state, { type }) => state.getIn(['timelines', type, 'items'], ImmutableList()),
(state) => state.get('statuses'),
], (columnSettings, statusIds, statuses) => {
const rawRegex = columnSettings.getIn(['regex', 'body'], '').trim();
let regex = null;
try {
regex = rawRegex && new RegExp(rawRegex, 'i');
} catch (e) {
// Bad regex, don't affect filters
}
return statusIds.filter(id => {
if (id === null) return true;
......@@ -34,11 +25,6 @@ const makeGetStatusIds = () => createSelector([
showStatus = showStatus && (statusForId.get('in_reply_to_id') === null || statusForId.get('in_reply_to_account_id') === me);
}
if (showStatus && regex && statusForId.get('account') !== me) {
const searchIndex = statusForId.get('reblog') ? statuses.getIn([statusForId.get('reblog'), 'search_index']) : statusForId.get('search_index');
showStatus = !regex.test(searchIndex);
}
return showStatus;
});
});
......
......@@ -12,6 +12,7 @@ import { debounce } from 'lodash';
import { uploadCompose, resetCompose } from '../../actions/compose';
import { expandHomeTimeline } from '../../actions/timelines';
import { expandNotifications } from '../../actions/notifications';
import { fetchFilters } from '../../actions/filters';
import { clearHeight } from '../../actions/height_cache';
import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
import UploadArea from './components/upload_area';
......@@ -297,6 +298,7 @@ export default class UI extends React.PureComponent {
this.props.dispatch(expandHomeTimeline());
this.props.dispatch(expandNotifications());
setTimeout(() => this.props.dispatch(fetchFilters()), 500);
}
componentDidMount () {
......
import { FILTERS_FETCH_SUCCESS } from '../actions/filters';
import { List as ImmutableList, fromJS } from 'immutable';
export default function filters(state = ImmutableList(), action) {
switch(action.type) {
case FILTERS_FETCH_SUCCESS:
return fromJS(action.filters);
default:
return state;
}
};
......@@ -26,6 +26,7 @@ import height_cache from './height_cache';
import custom_emojis from './custom_emojis';
import lists from './lists';
import listEditor from './list_editor';
import filters from './filters';
const reducers = {
dropdown_menu,
......@@ -55,6 +56,7 @@ const reducers = {
custom_emojis,
lists,
listEditor,
filters,
};
export default combineReducers(reducers);
......@@ -19,16 +19,44 @@ export const makeGetAccount = () => {
});
};
const toServerSideType = columnType => {
switch (columnType) {
case 'home':
case 'notifications':
case 'public':
case 'thread':
return columnType;
default:
if (columnType.indexOf('list:') > -1) {
return 'home';
} else {
return 'public'; // community, account, hashtag
}
}
};
const escapeRegExp = string =>
string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
const regexFromFilters = filters => {
if (filters.size === 0) {
return null;
}
return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).join('|'), 'i');
};
export const makeGetStatus = () => {
return createSelector(
[
(state, id) => state.getIn(['statuses', id]),
(state, id) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]),
(state, id) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
(state, id) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
(state, { id }) => state.getIn(['statuses', id]),
(state, { id }) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]),
(state, { id }) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
(state, { id }) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
(state, { contextType }) => state.get('filters', ImmutableList()).filter(filter => contextType && filter.get('context').includes(toServerSideType(contextType)) && (filter.get('expires_at') === null || Date.parse(filter.get('expires_at')) > (new Date()))),
],
(statusBase, statusReblog, accountBase, accountReblog) => {
(statusBase, statusReblog, accountBase, accountReblog, filters) => {
if (!statusBase) {
return null;
}
......@@ -39,9 +67,13 @@ export const makeGetStatus = () => {
statusReblog = null;
}
const regex = regexFromFilters(filters);
const filtered = regex && regex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index'));
return statusBase.withMutations(map => {
map.set('reblog', statusReblog);
map.set('account', accountBase);
map.set('filtered', filtered);
});
}
);
......
......@@ -725,6 +725,20 @@
vertical-align: middle;
}
.status__wrapper--filtered {
color: $dark-text-color;
border: 0;
font-size: inherit;
text-align: center;
line-height: inherit;
margin: 0;
padding: 15px;
box-sizing: border-box;
width: 100%;
clear: both;
border-bottom: 1px solid lighten($ui-base-color, 8%);
}
.status__prepend-icon-wrapper {
left: -26px;
position: absolute;
......
......@@ -153,6 +153,7 @@ class FeedManager
def filter_from_home?(status, receiver_id)
return false if receiver_id == status.account_id
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
return true if phrase_filtered?(status, receiver_id, :home)
check_for_blocks = status.mentions.pluck(:account_id)
check_for_blocks.concat([status.account_id])
......@@ -177,6 +178,7 @@ class FeedManager
def filter_from_mentions?(status, receiver_id)
return true if receiver_id == status.account_id
return true if phrase_filtered?(status, receiver_id, :notifications)
# This filter is called from NotifyService, but already after the sender of
# the notification has been checked for mute/block. Therefore, it's not
......@@ -190,6 +192,20 @@ class FeedManager
should_filter
end
def phrase_filtered?(status, receiver_id, context)
active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a
active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? }
active_filters.map! { |filter| Regexp.new(Regexp.escape(filter.phrase), true) }
return false if active_filters.empty?
combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) }
!combined_regex.match(status.text).nil? ||
(status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?)
end
# Adds a status to an account's feed, returning true if a status was
# added, and false if it was not added to the feed. Note that this is
# an internal helper: callers must call trim or push updates if
......
......@@ -99,6 +99,7 @@ class Account < ApplicationRecord
has_many :targeted_reports, class_name: 'Report', foreign_key: :target_account_id
has_many :report_notes, dependent: :destroy
has_many :custom_filters, inverse_of: :account, dependent: :destroy
# Moderation notes
has_many :account_moderation_notes, dependent: :destroy
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment