Commit ca3745e2 authored by Angelina Filippova's avatar Angelina Filippova

Merge branch 'feature/cache-invalidation' into 'develop'

Add ability to evict and ban URLs from the Pleroma MediaProxy cache

Closes #122

See merge request !142
parents 7a6b9f2e fdb2b6d2
Pipeline #29116 passed with stages
in 8 minutes and 26 seconds
......@@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Support pagination of local emoji packs and files
- Add MRF Activity Expiration setting
- Add ability to disable multi-factor authentication for a user
- Add ability to manually evict and ban URLs from the Pleroma MediaProxy cache
- Add Invalidation settings on MediaProxy tab
- Ability to configure S3 settings on Upload tab
- Show number of open reports in Sidebar Menu
- Add confirmation message when deleting a user
......@@ -30,6 +32,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Remove ability to moderate users that don't have valid nickname
- Displays both labels and description in the header of group of settiings
- Ability to add custom values in Pleroma.Upload.Filter.Mogrify setting
- Change types of the following settings: ':groups', ':replace', ':federated_timeline_removal', ':reject', ':match_actor'. Update functions that parses and wraps settings data according to this change.
- Move rendering Crontab setting from a separate component to EditableKeyword component
- Show only those MRF settings that have been enabled in MRF Policies setting
- Move Auto Linker settings to Link Formatter Tab as its configuration was moved to :pleroma, Pleroma.Formatter
......
import request from '@/utils/request'
import { getToken } from '@/utils/auth'
import { baseName } from './utils'
export async function listBannedUrls(page, authHost, token) {
return await request({
baseURL: baseName(authHost),
url: `/api/pleroma/admin/media_proxy_caches?page=${page}`,
method: 'get',
headers: authHeaders(token)
})
}
export async function purgeUrls(urls, ban, authHost, token) {
return await request({
baseURL: baseName(authHost),
url: `/api/pleroma/admin/media_proxy_caches/purge`,
method: 'post',
headers: authHeaders(token),
data: { urls, ban }
})
}
export async function removeBannedUrls(urls, authHost, token) {
return await request({
baseURL: baseName(authHost),
url: `/api/pleroma/admin/media_proxy_caches/delete`,
method: 'post',
headers: authHeaders(token),
data: { urls }
})
}
const authHeaders = (token) => token ? { 'Authorization': `Bearer ${getToken()}` } : {}
......@@ -65,8 +65,11 @@ export default {
externalLink: 'External Link',
users: 'Users',
reports: 'Reports',
invites: 'Invites',
statuses: 'Statuses',
settings: 'Settings',
moderationLog: 'Moderation Log',
mediaProxyCache: 'MediaProxy Cache',
'emoji-packs': 'Emoji packs'
},
navbar: {
......@@ -89,6 +92,19 @@ export default {
pleromaFELoginFailed: 'Failed to login via PleromaFE, please login with username/password',
pleromaFELoginSucceed: 'Logged in via PleromaFE'
},
mediaProxyCache: {
mediaProxyCache: 'MediaProxy Cache',
ban: 'Ban',
url: 'URL',
evict: 'Evict',
evictedMessage: 'This URL was evicted',
actions: 'Actions',
remove: 'Remove from Cachex',
evictObjectsHeader: 'Evict object from the MediaProxy cache',
listBannedUrlsHeader: 'List of all banned MediaProxy URLs',
multipleInput: 'You can enter a single URL or several comma separated links',
removeSelected: 'Remove Selected'
},
documentation: {
documentation: 'Documentation',
github: 'Github Repository'
......
......@@ -16,7 +16,7 @@ const settings = {
path: 'index',
component: () => import('@/views/settings/index'),
name: 'Settings',
meta: { title: 'Settings', icon: 'settings', noCache: true }
meta: { title: 'settings', icon: 'settings', noCache: true }
}
]
}
......@@ -30,7 +30,7 @@ const statuses = {
path: 'index',
component: () => import('@/views/statuses/index'),
name: 'Statuses',
meta: { title: 'Statuses', icon: 'form', noCache: true }
meta: { title: 'statuses', icon: 'form', noCache: true }
}
]
}
......@@ -44,7 +44,7 @@ const reports = {
path: 'index',
component: () => import('@/views/reports/index'),
name: 'Reports',
meta: { title: 'Reports', icon: 'documentation', noCache: true }
meta: { title: 'reports', icon: 'documentation', noCache: true }
}
]
}
......@@ -58,7 +58,7 @@ const invites = {
path: 'index',
component: () => import('@/views/invites/index'),
name: 'Invites',
meta: { title: 'Invites', icon: 'guide', noCache: true }
meta: { title: 'invites', icon: 'guide', noCache: true }
}
]
}
......@@ -72,7 +72,7 @@ const emojiPacks = {
path: 'index',
component: () => import('@/views/emojiPacks/index'),
name: 'Emoji Packs',
meta: { title: 'Emoji Packs', icon: 'eye-open', noCache: true }
meta: { title: 'emoji-packs', icon: 'eye-open', noCache: true }
}
]
}
......@@ -91,6 +91,20 @@ const moderationLog = {
]
}
const mediaProxyCacheDisabled = disabledFeatures.includes('media-proxy-cache')
const mediaProxyCache = {
path: '/media_proxy_cache',
component: Layout,
children: [
{
path: 'index',
component: () => import('@/views/mediaProxyCache/index'),
name: 'MediaProxy Cache',
meta: { title: 'mediaProxyCache', icon: 'example', noCache: true }
}
]
}
export const constantRouterMap = [
{
path: '/redirect',
......@@ -159,6 +173,7 @@ export const asyncRouterMap = [
...(invitesDisabled ? [] : [invites]),
...(emojiPacksDisabled ? [] : [emojiPacks]),
...(moderationLogDisabled ? [] : [moderationLog]),
...(mediaProxyCacheDisabled ? [] : [mediaProxyCache]),
...(settingsDisabled ? [] : [settings]),
{
path: '/users/:id',
......
......@@ -5,6 +5,7 @@ import emojiPacks from './modules/emojiPacks'
import errorLog from './modules/errorLog'
import getters from './getters'
import invites from './modules/invites'
import mediaProxyCache from './modules/mediaProxyCache'
import moderationLog from './modules/moderationLog'
import peers from './modules/peers'
import permission from './modules/permission'
......@@ -24,8 +25,9 @@ const store = new Vuex.Store({
app,
errorLog,
emojiPacks,
moderationLog,
invites,
mediaProxyCache,
moderationLog,
peers,
permission,
relays,
......
import { listBannedUrls, purgeUrls, removeBannedUrls } from '@/api/mediaProxyCache'
import { Message } from 'element-ui'
import i18n from '@/lang'
const mediaProxyCache = {
state: {
bannedUrls: [],
bannedUrlsCount: 0,
currentPage: 1,
loading: false
},
mutations: {
SET_BANNED_URLS: (state, urls) => {
state.bannedUrls = urls.map(el => { return { url: el } })
},
SET_BANNED_URLS_COUNT: (state, count) => {
state.bannedUrlsCount = count
},
SET_LOADING: (state, status) => {
state.loading = status
},
SET_PAGE: (state, page) => {
state.currentPage = page
}
},
actions: {
async ListBannedUrls({ commit, getters }, page) {
commit('SET_LOADING', true)
const response = await listBannedUrls(page, getters.authHost, getters.token)
commit('SET_BANNED_URLS', response.data.urls)
// commit('SET_BANNED_URLS_COUNT', count)
commit('SET_PAGE', page)
commit('SET_LOADING', false)
},
async PurgeUrls({ dispatch, getters, state }, { urls, ban }) {
await purgeUrls(urls, ban, getters.authHost, getters.token)
Message({
message: i18n.t('mediaProxyCache.evictedMessage'),
type: 'success',
duration: 5 * 1000
})
if (ban) {
dispatch('ListBannedUrls', state.currentPage)
}
},
async RemoveBannedUrls({ dispatch, getters, state }, urls) {
await removeBannedUrls(urls, getters.authHost, getters.token)
dispatch('ListBannedUrls', state.currentPage)
}
}
}
export default mediaProxyCache
......@@ -71,18 +71,16 @@ export const parseTuples = (tuples, key) => {
return [...acc, { [mascot.tuple[0]]: { ...mascot.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
} else if (Array.isArray(item.tuple[1]) &&
(item.tuple[0] === ':groups' || item.tuple[0] === ':replace' || item.tuple[0] === ':retries')) {
accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
return [...acc, { [group.tuple[0]]: { value: group.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
} else if (item.tuple[0] === ':crontab') {
accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
return { ...acc, [group.tuple[1]]: group.tuple[0] }
}, {})
} else if (item.tuple[0] === ':match_actor') {
accum[item.tuple[0]] = Object.keys(item.tuple[1]).reduce((acc, regex) => {
return [...acc, { [regex]: { value: item.tuple[1][regex], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
(item.tuple[0] === ':groups' || item.tuple[0] === ':replace' || item.tuple[0] === ':retries' || item.tuple[0] === ':headers' || item.tuple[0] === ':crontab')) {
if (item.tuple[0] === ':crontab') {
accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
return [...acc, { [group.tuple[1]]: { value: group.tuple[0], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
} else {
accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
return [...acc, { [group.tuple[0]]: { value: group.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
}
} else if (item.tuple[0] === ':icons') {
accum[item.tuple[0]] = item.tuple[1].map(icon => {
return Object.keys(icon).map(name => {
......@@ -103,7 +101,13 @@ export const parseTuples = (tuples, key) => {
} else if (item.tuple[0] === ':ip') {
accum[item.tuple[0]] = item.tuple[1].tuple.join('.')
} else if (item.tuple[1] && typeof item.tuple[1] === 'object') {
accum[item.tuple[0]] = parseObject(item.tuple[1])
if (item.tuple[0] === ':params' || item.tuple[0] === ':match_actor') {
accum[item.tuple[0]] = Object.keys(item.tuple[1]).reduce((acc, key) => {
return [...acc, { [key]: { value: item.tuple[1][key], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
}, [])
} else {
accum[item.tuple[0]] = parseObject(item.tuple[1])
}
} else {
accum[item.tuple[0]] = item.tuple[1]
}
......@@ -214,11 +218,11 @@ export const wrapUpdatedSettings = (group, settings, currentState) => {
const wrapValues = (settings, currentState) => {
return Object.keys(settings).map(setting => {
const [type, value] = settings[setting]
if (
type === 'keyword' ||
type.includes('keyword') ||
type.includes('tuple') && type.includes('list') ||
setting === ':replace'
if (type === 'keyword' ||
(Array.isArray(type) && (
type.includes('keyword') ||
(type.includes('tuple') && type.includes('list'))
))
) {
return { 'tuple': [setting, wrapValues(value, currentState)] }
} else if (prependWithСolon(type, value)) {
......@@ -231,15 +235,16 @@ const wrapValues = (settings, currentState) => {
return { 'tuple': [value, setting] }
} else if (type === 'map') {
const mapValue = Object.keys(value).reduce((acc, key) => {
acc[key] = setting === ':match_actor' ? value[key] : value[key][1]
acc[key] = value[key][1]
return acc
}, {})
return { 'tuple': [setting, { ...currentState[setting], ...mapValue }] }
} else if (type.includes('map')) {
const mapValue = Object.keys(value).reduce((acc, key) => {
acc[key] = value[key][1]
return acc
}, {})
const mapCurrentState = setting === ':match_actor'
? currentState[setting].reduce((acc, element) => {
return { ...acc, ...{ [Object.keys(element)[0]]: Object.values(element)[0].value }}
}, {})
: currentState[setting]
return { 'tuple': [setting, { ...mapCurrentState, ...mapValue }] }
return { 'tuple': [setting, mapValue] }
} else if (setting === ':ip') {
const ip = value.split('.').map(s => parseInt(s, 10))
return { 'tuple': [setting, { 'tuple': ip }] }
......
<template>
<div class="media-proxy-cache-container">
<div class="media-proxy-cache-header-container">
<h1>{{ $t('mediaProxyCache.mediaProxyCache') }}</h1>
<reboot-button/>
</div>
<p class="media-proxy-cache-header">{{ $t('mediaProxyCache.evictObjectsHeader') }}</p>
<div class="url-input-container">
<el-input
:placeholder="$t('mediaProxyCache.url')"
v-model="urls"
type="textarea"
autosize
clearable
class="url-input"/>
<el-checkbox v-model="ban">{{ $t('mediaProxyCache.ban') }}</el-checkbox>
<el-button class="evict-button" @click="evictURL">{{ $t('mediaProxyCache.evict') }}</el-button>
</div>
<span class="expl url-input-expl">{{ $t('mediaProxyCache.multipleInput') }}</span>
<p class="media-proxy-cache-header">{{ $t('mediaProxyCache.listBannedUrlsHeader') }}</p>
<el-table
v-loading="loading"
:data="bannedUrls"
class="banned-urls-table"
@selection-change="handleSelectionChange">>
<el-table-column
type="selection"
align="center"
width="55"/>
<el-table-column
:min-width="isDesktop ? 320 : 120"
prop="url"/>
<el-table-column>
<template slot="header">
<el-button
:disabled="removeSelectedDisabled"
size="mini"
class="remove-url-button"
@click="removeSelected()">{{ $t('mediaProxyCache.removeSelected') }}</el-button>
</template>
<template slot-scope="scope">
<el-button
size="mini"
class="remove-url-button"
@click="removeUrl(scope.row.url)">{{ $t('mediaProxyCache.remove') }}</el-button>
</template>
</el-table-column>
</el-table>
</div>
</template>
<script>
import RebootButton from '@/components/RebootButton'
export default {
name: 'MediaProxyCache',
components: { RebootButton },
data() {
return {
urls: '',
ban: false,
selectedUrls: []
}
},
computed: {
bannedUrls() {
return this.$store.state.mediaProxyCache.bannedUrls
},
isDesktop() {
return this.$store.state.app.device === 'desktop'
},
loading() {
return this.$store.state.mediaProxyCache.loading
},
removeSelectedDisabled() {
return this.selectedUrls.length === 0
}
},
mounted() {
this.$store.dispatch('GetNodeInfo')
this.$store.dispatch('NeedReboot')
this.$store.dispatch('ListBannedUrls', 1)
},
methods: {
evictURL() {
const urls = this.urls.split(',').map(url => url.trim()).filter(el => el.length > 0)
this.$store.dispatch('PurgeUrls', { urls, ban: this.ban })
this.urls = ''
},
handleSelectionChange(value) {
this.$data.selectedUrls = value
},
removeSelected() {
const urls = this.selectedUrls.map(el => el.url)
this.$store.dispatch('RemoveBannedUrls', urls)
this.selectedUrls = []
},
removeUrl(url) {
this.$store.dispatch('RemoveBannedUrls', [url])
}
}
}
</script>
<style rel='stylesheet/scss' lang='scss' scoped>
h1 {
margin: 0;
}
.expl {
color: #666666;
font-size: 13px;
line-height: 22px;
margin: 5px 0 0 0;
overflow-wrap: break-word;
overflow: hidden;
text-overflow: ellipsis;
}
.banned-urls-table {
margin-top: 15px;
margin-bottom: 15px;
}
.evict-button {
margin-left: 15px;
}
.media-proxy-cache-header {
margin-left: 15px;
margin-top: 22px;
font-weight: 500;
}
.media-proxy-cache-header-container {
display: flex;
align-items: center;
justify-content: space-between;
margin: 10px 15px;
}
.remove-url-button {
width: 150px;
}
.url-input {
margin-right: 15px;
}
.url-input-container {
display: flex;
align-items: baseline;
margin: 15px 15px 5px 15px;
}
.url-input-expl {
margin-left: 15px;
}
@media only screen and (max-width:480px) {
.url-input {
width: 100%;
margin-bottom: 5px;
}
}
</style>
......@@ -95,15 +95,14 @@
<el-input
v-if="setting.type === 'atom'"
:value="inputValue"
:placeholder="setting.suggestions[0] ? setting.suggestions[0].substr(1) : ''"
:placeholder="setting.suggestions && setting.suggestions[0] ? setting.suggestions[0].substr(1) : ''"
:data-search="setting.key || setting.group"
class="input"
@input="update($event, settingGroup.group, settingGroup.key, settingParent, setting.key, setting.type, nested)">
<template slot="prepend">:</template>
</el-input>
<!-- special inputs -->
<crontab-input v-if="setting.key === ':crontab'" :data="data[setting.key]" :setting-group="settingGroup" :setting="setting"/>
<editable-keyword-input v-if="editableKeyword(setting.key, setting.type)" :data="keywordData" :setting-group="settingGroup" :setting="setting"/>
<editable-keyword-input v-if="editableKeyword(setting.key, setting.type)" :data="keywordData" :setting-group="settingGroup" :setting="setting" :parents="settingParent"/>
<icons-input v-if="setting.key === ':icons'" :data="iconsData" :setting-group="settingGroup" :setting="setting"/>
<link-formatter-input v-if="booleanCombinedInput" :data="data" :setting-group="settingGroup" :setting="setting"/>
<mascots-input v-if="setting.key === ':mascots'" :data="keywordData" :setting-group="settingGroup" :setting="setting"/>
......@@ -129,7 +128,6 @@
<script>
import i18n from '@/lang'
import {
CrontabInput,
EditableKeywordInput,
IconsInput,
ImageUploadInput,
......@@ -148,7 +146,6 @@ import marked from 'marked'
export default {
name: 'Inputs',
components: {
CrontabInput,
EditableKeywordInput,
IconsInput,
ImageUploadInput,
......@@ -225,7 +222,7 @@ export default {
this.$store.state.settings.db[group][key].includes(this.setting.key)
},
iconsData() {
return Array.isArray(this.data[':icons']) ? this.data[':icons'] : []
return Array.isArray(this.data) ? this.data : []
},
inputValue() {
if ([':esshd', ':cors_plug', ':quack', ':tesla', ':swoosh'].includes(this.settingGroup.group) &&
......@@ -267,6 +264,10 @@ export default {
}
},
keywordData() {
if (this.settingParent.length > 0 ||
(Array.isArray(this.setting.type) && this.setting.type.includes('tuple') && this.setting.type.includes('list'))) {
return Array.isArray(this.data[this.setting.key]) ? this.data[this.setting.key] : []
}
return Array.isArray(this.data) ? this.data : []
},
reducedSelects() {
......@@ -296,10 +297,14 @@ export default {
},
methods: {
editableKeyword(key, type) {
return key === ':replace' ||
type === 'map' ||
(Array.isArray(type) && type.includes('keyword') && type.includes('integer')) ||
(Array.isArray(type) && type.includes('keyword') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1)
return Array.isArray(type) && (
(type.includes('map') && type.includes('string')) ||
(type.includes('map') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1) ||
(type.includes('keyword') && type.includes('integer')) ||
(type.includes('keyword') && type.includes('string')) ||
(type.includes('tuple') && type.includes('list')) ||
(type.includes('keyword') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1)
)
},
getFormattedDescription(desc) {
return marked(desc)
......@@ -346,7 +351,7 @@ export default {
type.includes('module') ||
(type.includes('list') && type.includes('string')) ||
(type.includes('list') && type.includes('atom')) ||
(type.includes('regex') && type.includes('string'))
(!type.includes('keyword') && type.includes('regex') && type.includes('string'))
)
},
renderSingleSelect(type) {
......
......@@ -3,6 +3,14 @@
<el-form v-if="!loading" :model="mediaProxyData" :label-position="labelPosition" :label-width="labelWidth">
<setting :setting-group="mediaProxy" :data="mediaProxyData"/>
</el-form>
<el-divider v-if="mediaProxy" class="divider thick-line"/>
<el-form v-if="!loading" :model="httpInvalidationData" :label-position="labelPosition" :label-width="labelWidth">
<setting :setting-group="httpInvalidation" :data="httpInvalidationData"/>
</el-form>
<el-divider v-if="httpInvalidation" class="divider thick-line"/>
<el-form v-if="!loading" :model="scriptInvalidationData" :label-position="labelPosition" :label-width="labelWidth">
<setting :setting-group="scriptInvalidation" :data="scriptInvalidationData"/>
</el-form>
<div class="submit-button-container">
<el-button class="submit-button" type="primary" @click="onSubmit">Submit</el-button>
</div>
......@@ -22,6 +30,12 @@ export default {
...mapGetters([
'settings'
]),
httpInvalidation() {
return this.settings.description.find(setting => setting.key === 'Pleroma.Web.MediaProxy.Invalidation.Http')
},
httpInvalidationData() {
return _.get(this.settings.settings, [':pleroma', 'Pleroma.Web.MediaProxy.Invalidation.Http']) || {}
},
isMobile() {
return this.$store.state.app.device === 'mobile'
},
......@@ -51,6 +65,12 @@ export default {
},
mediaProxyData() {
return _.get(this.settings.settings, [':pleroma', ':media_proxy']) || {}
},
scriptInvalidation() {
return this.settings.description.find(setting => setting.key === 'Pleroma.Web.MediaProxy.Invalidation.Script')
},
scriptInvalidationData() {
return _.get(this.settings.settings, [':pleroma', 'Pleroma.Web.MediaProxy.Invalidation.Script']) || {}
}
},
methods: {
......
......@@ -122,7 +122,7 @@ export default {
return type === 'keyword' ||
type === 'map' ||
type.includes('keyword') ||
key === ':replace'
type.includes('map')
},
divideSetting(key) {
return [':sslopts', ':tlsopts', ':adapter', ':poll_limits', ':queues', ':styling', ':invalidation', ':multi_factor_authentication'].includes(key)
......
<template>
<el-form :label-width="labelWidth" :label-position="isMobile ? 'top' : 'right'" class="crontab">
<el-form-item v-for="worker in workers" :key="worker" :label="worker" :data-search="setting.key" class="crontab-container">
<el-input
:value="data[worker]"
:placeholder="getSuggestion(worker) || null"
class="input setting-input"