Merge branch 'feature/cache-invalidation' into 'develop'

Add ability to evict and ban URLs from the Pleroma MediaProxy cache

Closes #122

See merge request pleroma/admin-fe!142

CHANGELOG.md

@@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support pagination of local emoji packs and files
 - Add MRF Activity Expiration setting
 - Add ability to disable multi-factor authentication for a user
+- Add ability to manually evict and ban URLs from the Pleroma MediaProxy cache
+- Add Invalidation settings on MediaProxy tab
 - Ability to configure S3 settings on Upload tab
 - Show number of open reports in Sidebar Menu
 - Add confirmation message when deleting a user
@@ -30,6 +32,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Remove ability to moderate users that don't have valid nickname
 - Displays both labels and description in the header of group of settiings
 - Ability to add custom values in Pleroma.Upload.Filter.Mogrify setting
+- Change types of the following settings: ':groups', ':replace', ':federated_timeline_removal', ':reject', ':match_actor'. Update functions that parses and wraps settings data according to this change.
+- Move rendering Crontab setting from a separate component to EditableKeyword component
 - Show only those MRF settings that have been enabled in MRF Policies setting
 - Move Auto Linker settings to Link Formatter Tab as its configuration was moved to :pleroma, Pleroma.Formatter
 

src/api/mediaProxyCache.js

+import request from '@/utils/request'
+import { getToken } from '@/utils/auth'
+import { baseName } from './utils'
+
+export async function listBannedUrls(page, authHost, token) {
+  return await request({
+    baseURL: baseName(authHost),
+    url: `/api/pleroma/admin/media_proxy_caches?page=${page}`,
+    method: 'get',
+    headers: authHeaders(token)
+  })
+}
+
+export async function purgeUrls(urls, ban, authHost, token) {
+  return await request({
+    baseURL: baseName(authHost),
+    url: `/api/pleroma/admin/media_proxy_caches/purge`,
+    method: 'post',
+    headers: authHeaders(token),
+    data: { urls, ban }
+  })
+}
+
+export async function removeBannedUrls(urls, authHost, token) {
+  return await request({
+    baseURL: baseName(authHost),
+    url: `/api/pleroma/admin/media_proxy_caches/delete`,
+    method: 'post',
+    headers: authHeaders(token),
+    data: { urls }
+  })
+}
+
+const authHeaders = (token) => token ? { 'Authorization': `Bearer ${getToken()}` } : {}

src/lang/en.js

@@ -65,8 +65,11 @@ export default {
     externalLink: 'External Link',
     users: 'Users',
     reports: 'Reports',
+    invites: 'Invites',
+    statuses: 'Statuses',
     settings: 'Settings',
     moderationLog: 'Moderation Log',
+    mediaProxyCache: 'MediaProxy Cache',
     'emoji-packs': 'Emoji packs'
   },
   navbar: {
@@ -89,6 +92,19 @@ export default {
     pleromaFELoginFailed: 'Failed to login via PleromaFE, please login with username/password',
     pleromaFELoginSucceed: 'Logged in via PleromaFE'
   },
+  mediaProxyCache: {
+    mediaProxyCache: 'MediaProxy Cache',
+    ban: 'Ban',
+    url: 'URL',
+    evict: 'Evict',
+    evictedMessage: 'This URL was evicted',
+    actions: 'Actions',
+    remove: 'Remove from Cachex',
+    evictObjectsHeader: 'Evict object from the MediaProxy cache',
+    listBannedUrlsHeader: 'List of all banned MediaProxy URLs',
+    multipleInput: 'You can enter a single URL or several comma separated links',
+    removeSelected: 'Remove Selected'
+  },
   documentation: {
     documentation: 'Documentation',
     github: 'Github Repository'

src/router/index.js

@@ -16,7 +16,7 @@ const settings = {
       path: 'index',
       component: () => import('@/views/settings/index'),
       name: 'Settings',
-      meta: { title: 'Settings', icon: 'settings', noCache: true }
+      meta: { title: 'settings', icon: 'settings', noCache: true }
     }
   ]
 }
@@ -30,7 +30,7 @@ const statuses = {
       path: 'index',
       component: () => import('@/views/statuses/index'),
       name: 'Statuses',
-      meta: { title: 'Statuses', icon: 'form', noCache: true }
+      meta: { title: 'statuses', icon: 'form', noCache: true }
     }
   ]
 }
@@ -44,7 +44,7 @@ const reports = {
       path: 'index',
       component: () => import('@/views/reports/index'),
       name: 'Reports',
-      meta: { title: 'Reports', icon: 'documentation', noCache: true }
+      meta: { title: 'reports', icon: 'documentation', noCache: true }
     }
   ]
 }
@@ -58,7 +58,7 @@ const invites = {
       path: 'index',
       component: () => import('@/views/invites/index'),
       name: 'Invites',
-      meta: { title: 'Invites', icon: 'guide', noCache: true }
+      meta: { title: 'invites', icon: 'guide', noCache: true }
     }
   ]
 }
@@ -72,7 +72,7 @@ const emojiPacks = {
       path: 'index',
       component: () => import('@/views/emojiPacks/index'),
       name: 'Emoji Packs',
-      meta: { title: 'Emoji Packs', icon: 'eye-open', noCache: true }
+      meta: { title: 'emoji-packs', icon: 'eye-open', noCache: true }
     }
   ]
 }
@@ -91,6 +91,20 @@ const moderationLog = {
   ]
 }
 
+const mediaProxyCacheDisabled = disabledFeatures.includes('media-proxy-cache')
+const mediaProxyCache = {
+  path: '/media_proxy_cache',
+  component: Layout,
+  children: [
+    {
+      path: 'index',
+      component: () => import('@/views/mediaProxyCache/index'),
+      name: 'MediaProxy Cache',
+      meta: { title: 'mediaProxyCache', icon: 'example', noCache: true }
+    }
+  ]
+}
+
 export const constantRouterMap = [
   {
     path: '/redirect',
@@ -159,6 +173,7 @@ export const asyncRouterMap = [
   ...(invitesDisabled ? [] : [invites]),
   ...(emojiPacksDisabled ? [] : [emojiPacks]),
   ...(moderationLogDisabled ? [] : [moderationLog]),
+  ...(mediaProxyCacheDisabled ? [] : [mediaProxyCache]),
   ...(settingsDisabled ? [] : [settings]),
   {
     path: '/users/:id',

src/store/index.js

@@ -5,6 +5,7 @@ import emojiPacks from './modules/emojiPacks'
 import errorLog from './modules/errorLog'
 import getters from './getters'
 import invites from './modules/invites'
+import mediaProxyCache from './modules/mediaProxyCache'
 import moderationLog from './modules/moderationLog'
 import peers from './modules/peers'
 import permission from './modules/permission'
@@ -24,8 +25,9 @@ const store = new Vuex.Store({
     app,
     errorLog,
     emojiPacks,
-    moderationLog,
     invites,
+    mediaProxyCache,
+    moderationLog,
     peers,
     permission,
     relays,

src/store/modules/mediaProxyCache.js

+import { listBannedUrls, purgeUrls, removeBannedUrls } from '@/api/mediaProxyCache'
+import { Message } from 'element-ui'
+import i18n from '@/lang'
+
+const mediaProxyCache = {
+  state: {
+    bannedUrls: [],
+    bannedUrlsCount: 0,
+    currentPage: 1,
+    loading: false
+  },
+  mutations: {
+    SET_BANNED_URLS: (state, urls) => {
+      state.bannedUrls = urls.map(el => { return { url: el } })
+    },
+    SET_BANNED_URLS_COUNT: (state, count) => {
+      state.bannedUrlsCount = count
+    },
+    SET_LOADING: (state, status) => {
+      state.loading = status
+    },
+    SET_PAGE: (state, page) => {
+      state.currentPage = page
+    }
+  },
+  actions: {
+    async ListBannedUrls({ commit, getters }, page) {
+      commit('SET_LOADING', true)
+      const response = await listBannedUrls(page, getters.authHost, getters.token)
+      commit('SET_BANNED_URLS', response.data.urls)
+      // commit('SET_BANNED_URLS_COUNT', count)
+      commit('SET_PAGE', page)
+      commit('SET_LOADING', false)
+    },
+    async PurgeUrls({ dispatch, getters, state }, { urls, ban }) {
+      await purgeUrls(urls, ban, getters.authHost, getters.token)
+      Message({
+        message: i18n.t('mediaProxyCache.evictedMessage'),
+        type: 'success',
+        duration: 5 * 1000
+      })
+      if (ban) {
+        dispatch('ListBannedUrls', state.currentPage)
+      }
+    },
+    async RemoveBannedUrls({ dispatch, getters, state }, urls) {
+      await removeBannedUrls(urls, getters.authHost, getters.token)
+      dispatch('ListBannedUrls', state.currentPage)
+    }
+  }
+}
+
+export default mediaProxyCache

src/store/modules/normalizers.js

@@ -71,18 +71,16 @@ export const parseTuples = (tuples, key) => {
         return [...acc, { [mascot.tuple[0]]: { ...mascot.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
       }, [])
     } else if (Array.isArray(item.tuple[1]) &&
-      (item.tuple[0] === ':groups' || item.tuple[0] === ':replace' || item.tuple[0] === ':retries')) {
-      accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
-        return [...acc, { [group.tuple[0]]: { value: group.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
-      }, [])
-    } else if (item.tuple[0] === ':crontab') {
-      accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
-        return { ...acc, [group.tuple[1]]: group.tuple[0] }
-      }, {})
-    } else if (item.tuple[0] === ':match_actor') {
-      accum[item.tuple[0]] = Object.keys(item.tuple[1]).reduce((acc, regex) => {
-        return [...acc, { [regex]: { value: item.tuple[1][regex], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
-      }, [])
+      (item.tuple[0] === ':groups' || item.tuple[0] === ':replace' || item.tuple[0] === ':retries' || item.tuple[0] === ':headers' || item.tuple[0] === ':crontab')) {
+      if (item.tuple[0] === ':crontab') {
+        accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
+          return [...acc, { [group.tuple[1]]: { value: group.tuple[0], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
+        }, [])
+      } else {
+        accum[item.tuple[0]] = item.tuple[1].reduce((acc, group) => {
+          return [...acc, { [group.tuple[0]]: { value: group.tuple[1], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
+        }, [])
+      }
     } else if (item.tuple[0] === ':icons') {
       accum[item.tuple[0]] = item.tuple[1].map(icon => {
         return Object.keys(icon).map(name => {
@@ -103,7 +101,13 @@ export const parseTuples = (tuples, key) => {
     } else if (item.tuple[0] === ':ip') {
       accum[item.tuple[0]] = item.tuple[1].tuple.join('.')
     } else if (item.tuple[1] && typeof item.tuple[1] === 'object') {
-      accum[item.tuple[0]] = parseObject(item.tuple[1])
+      if (item.tuple[0] === ':params' || item.tuple[0] === ':match_actor') {
+        accum[item.tuple[0]] = Object.keys(item.tuple[1]).reduce((acc, key) => {
+          return [...acc, { [key]: { value: item.tuple[1][key], id: `f${(~~(Math.random() * 1e8)).toString(16)}` }}]
+        }, [])
+      } else {
+        accum[item.tuple[0]] = parseObject(item.tuple[1])
+      }
     } else {
       accum[item.tuple[0]] = item.tuple[1]
     }
@@ -214,11 +218,11 @@ export const wrapUpdatedSettings = (group, settings, currentState) => {
 const wrapValues = (settings, currentState) => {
   return Object.keys(settings).map(setting => {
     const [type, value] = settings[setting]
-    if (
-      type === 'keyword' ||
-      type.includes('keyword') ||
-      type.includes('tuple') && type.includes('list') ||
-      setting === ':replace'
+    if (type === 'keyword' ||
+      (Array.isArray(type) && (
+        type.includes('keyword') ||
+        (type.includes('tuple') && type.includes('list'))
+      ))
     ) {
       return { 'tuple': [setting, wrapValues(value, currentState)] }
     } else if (prependWithСolon(type, value)) {
@@ -231,15 +235,16 @@ const wrapValues = (settings, currentState) => {
       return { 'tuple': [value, setting] }
     } else if (type === 'map') {
       const mapValue = Object.keys(value).reduce((acc, key) => {
-        acc[key] = setting === ':match_actor' ? value[key] : value[key][1]
+        acc[key] = value[key][1]
+        return acc
+      }, {})
+      return { 'tuple': [setting, { ...currentState[setting], ...mapValue }] }
+    } else if (type.includes('map')) {
+      const mapValue = Object.keys(value).reduce((acc, key) => {
+        acc[key] = value[key][1]
         return acc
       }, {})
-      const mapCurrentState = setting === ':match_actor'
-        ? currentState[setting].reduce((acc, element) => {
-          return { ...acc, ...{ [Object.keys(element)[0]]: Object.values(element)[0].value }}
-        }, {})
-        : currentState[setting]
-      return { 'tuple': [setting, { ...mapCurrentState, ...mapValue }] }
+      return { 'tuple': [setting, mapValue] }
     } else if (setting === ':ip') {
       const ip = value.split('.').map(s => parseInt(s, 10))
       return { 'tuple': [setting, { 'tuple': ip }] }

src/views/mediaProxyCache/index.vue

+<template>
+  <div class="media-proxy-cache-container">
+    <div class="media-proxy-cache-header-container">
+      <h1>{{ $t('mediaProxyCache.mediaProxyCache') }}</h1>
+      <reboot-button/>
+    </div>
+    <p class="media-proxy-cache-header">{{ $t('mediaProxyCache.evictObjectsHeader') }}</p>
+    <div class="url-input-container">
+      <el-input
+        :placeholder="$t('mediaProxyCache.url')"
+        v-model="urls"
+        type="textarea"
+        autosize
+        clearable
+        class="url-input"/>
+      <el-checkbox v-model="ban">{{ $t('mediaProxyCache.ban') }}</el-checkbox>
+      <el-button class="evict-button" @click="evictURL">{{ $t('mediaProxyCache.evict') }}</el-button>
+    </div>
+    <span class="expl url-input-expl">{{ $t('mediaProxyCache.multipleInput') }}</span>
+    <p class="media-proxy-cache-header">{{ $t('mediaProxyCache.listBannedUrlsHeader') }}</p>
+    <el-table
+      v-loading="loading"
+      :data="bannedUrls"
+      class="banned-urls-table"
+      @selection-change="handleSelectionChange">>
+      <el-table-column
+        type="selection"
+        align="center"
+        width="55"/>
+      <el-table-column
+        :min-width="isDesktop ? 320 : 120"
+        prop="url"/>
+      <el-table-column>
+        <template slot="header">
+          <el-button
+            :disabled="removeSelectedDisabled"
+            size="mini"
+            class="remove-url-button"
+            @click="removeSelected()">{{ $t('mediaProxyCache.removeSelected') }}</el-button>
+        </template>
+        <template slot-scope="scope">
+          <el-button
+            size="mini"
+            class="remove-url-button"
+            @click="removeUrl(scope.row.url)">{{ $t('mediaProxyCache.remove') }}</el-button>
+        </template>
+      </el-table-column>
+    </el-table>
+  </div>
+</template>
+
+<script>
+import RebootButton from '@/components/RebootButton'
+
+export default {
+  name: 'MediaProxyCache',
+  components: { RebootButton },
+  data() {
+    return {
+      urls: '',
+      ban: false,
+      selectedUrls: []
+    }
+  },
+  computed: {
+    bannedUrls() {
+      return this.$store.state.mediaProxyCache.bannedUrls
+    },
+    isDesktop() {
+      return this.$store.state.app.device === 'desktop'
+    },
+    loading() {
+      return this.$store.state.mediaProxyCache.loading
+    },
+    removeSelectedDisabled() {
+      return this.selectedUrls.length === 0
+    }
+  },
+  mounted() {
+    this.$store.dispatch('GetNodeInfo')
+    this.$store.dispatch('NeedReboot')
+    this.$store.dispatch('ListBannedUrls', 1)
+  },
+  methods: {
+    evictURL() {
+      const urls = this.urls.split(',').map(url => url.trim()).filter(el => el.length > 0)
+      this.$store.dispatch('PurgeUrls', { urls, ban: this.ban })
+      this.urls = ''
+    },
+    handleSelectionChange(value) {
+      this.$data.selectedUrls = value
+    },
+    removeSelected() {
+      const urls = this.selectedUrls.map(el => el.url)
+      this.$store.dispatch('RemoveBannedUrls', urls)
+      this.selectedUrls = []
+    },
+    removeUrl(url) {
+      this.$store.dispatch('RemoveBannedUrls', [url])
+    }
+  }
+}
+</script>
+
+<style rel='stylesheet/scss' lang='scss' scoped>
+h1 {
+  margin: 0;
+}
+.expl {
+  color: #666666;
+  font-size: 13px;
+  line-height: 22px;
+  margin: 5px 0 0 0;
+  overflow-wrap: break-word;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+.banned-urls-table {
+  margin-top: 15px;
+  margin-bottom: 15px;
+}
+.evict-button {
+  margin-left: 15px;
+}
+.media-proxy-cache-header {
+  margin-left: 15px;
+  margin-top: 22px;
+  font-weight: 500;
+}
+.media-proxy-cache-header-container {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  margin: 10px 15px;
+}
+.remove-url-button {
+  width: 150px;
+}
+.url-input {
+  margin-right: 15px;
+}
+.url-input-container {
+  display: flex;
+  align-items: baseline;
+  margin: 15px 15px 5px 15px;
+}
+.url-input-expl {
+  margin-left: 15px;
+}
+
+@media only screen and (max-width:480px) {
+  .url-input {
+    width: 100%;
+    margin-bottom: 5px;
+  }
+}
+</style>

src/views/settings/components/Inputs.vue

@@ -95,15 +95,14 @@
         <el-input
           v-if="setting.type === 'atom'"
           :value="inputValue"
-          :placeholder="setting.suggestions[0] ? setting.suggestions[0].substr(1) : ''"
+          :placeholder="setting.suggestions && setting.suggestions[0] ? setting.suggestions[0].substr(1) : ''"
           :data-search="setting.key || setting.group"
           class="input"
           @input="update($event, settingGroup.group, settingGroup.key, settingParent, setting.key, setting.type, nested)">
           <template slot="prepend">:</template>
         </el-input>
         <!-- special inputs -->
-        <crontab-input v-if="setting.key === ':crontab'" :data="data[setting.key]" :setting-group="settingGroup" :setting="setting"/>
-        <editable-keyword-input v-if="editableKeyword(setting.key, setting.type)" :data="keywordData" :setting-group="settingGroup" :setting="setting"/>
+        <editable-keyword-input v-if="editableKeyword(setting.key, setting.type)" :data="keywordData" :setting-group="settingGroup" :setting="setting" :parents="settingParent"/>
         <icons-input v-if="setting.key === ':icons'" :data="iconsData" :setting-group="settingGroup" :setting="setting"/>
         <link-formatter-input v-if="booleanCombinedInput" :data="data" :setting-group="settingGroup" :setting="setting"/>
         <mascots-input v-if="setting.key === ':mascots'" :data="keywordData" :setting-group="settingGroup" :setting="setting"/>
@@ -129,7 +128,6 @@
 <script>
 import i18n from '@/lang'
 import {
-  CrontabInput,
   EditableKeywordInput,
   IconsInput,
   ImageUploadInput,
@@ -148,7 +146,6 @@ import marked from 'marked'
 export default {
   name: 'Inputs',
   components: {
-    CrontabInput,
     EditableKeywordInput,
     IconsInput,
     ImageUploadInput,
@@ -225,7 +222,7 @@ export default {
         this.$store.state.settings.db[group][key].includes(this.setting.key)
     },
     iconsData() {
-      return Array.isArray(this.data[':icons']) ? this.data[':icons'] : []
+      return Array.isArray(this.data) ? this.data : []
     },
     inputValue() {
       if ([':esshd', ':cors_plug', ':quack', ':tesla', ':swoosh'].includes(this.settingGroup.group) &&
@@ -267,6 +264,10 @@ export default {
       }
     },
     keywordData() {
+      if (this.settingParent.length > 0 ||
+        (Array.isArray(this.setting.type) && this.setting.type.includes('tuple') && this.setting.type.includes('list'))) {
+        return Array.isArray(this.data[this.setting.key]) ? this.data[this.setting.key] : []
+      }
       return Array.isArray(this.data) ? this.data : []
     },
     reducedSelects() {
@@ -296,10 +297,14 @@ export default {
   },
   methods: {
     editableKeyword(key, type) {
-      return key === ':replace' ||
-        type === 'map' ||
-        (Array.isArray(type) && type.includes('keyword') && type.includes('integer')) ||
-        (Array.isArray(type) && type.includes('keyword') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1)
+      return Array.isArray(type) && (
+        (type.includes('map') && type.includes('string')) ||
+        (type.includes('map') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1) ||
+        (type.includes('keyword') && type.includes('integer')) ||
+        (type.includes('keyword') && type.includes('string')) ||
+        (type.includes('tuple') && type.includes('list')) ||
+        (type.includes('keyword') && type.findIndex(el => el.includes('list') && el.includes('string')) !== -1)
+      )
     },
     getFormattedDescription(desc) {
       return marked(desc)
@@ -346,7 +351,7 @@ export default {
         type.includes('module') ||
         (type.includes('list') && type.includes('string')) ||
         (type.includes('list') && type.includes('atom')) ||
-        (type.includes('regex') && type.includes('string'))
+        (!type.includes('keyword') && type.includes('regex') && type.includes('string'))
       )
     },
     renderSingleSelect(type) {

src/views/settings/components/MediaProxy.vue

@@ -3,6 +3,14 @@
     <el-form v-if="!loading" :model="mediaProxyData" :label-position="labelPosition" :label-width="labelWidth">
       <setting :setting-group="mediaProxy" :data="mediaProxyData"/>
     </el-form>
+    <el-divider v-if="mediaProxy" class="divider thick-line"/>
+    <el-form v-if="!loading" :model="httpInvalidationData" :label-position="labelPosition" :label-width="labelWidth">
+      <setting :setting-group="httpInvalidation" :data="httpInvalidationData"/>
+    </el-form>
+    <el-divider v-if="httpInvalidation" class="divider thick-line"/>
+    <el-form v-if="!loading" :model="scriptInvalidationData" :label-position="labelPosition" :label-width="labelWidth">
+      <setting :setting-group="scriptInvalidation" :data="scriptInvalidationData"/>
+    </el-form>
     <div class="submit-button-container">
       <el-button class="submit-button" type="primary" @click="onSubmit">Submit</el-button>
     </div>
@@ -22,6 +30,12 @@ export default {
     ...mapGetters([
       'settings'
     ]),
+    httpInvalidation() {
+      return this.settings.description.find(setting => setting.key === 'Pleroma.Web.MediaProxy.Invalidation.Http')
+    },
+    httpInvalidationData() {
+      return _.get(this.settings.settings, [':pleroma', 'Pleroma.Web.MediaProxy.Invalidation.Http']) || {}
+    },
     isMobile() {
       return this.$store.state.app.device === 'mobile'
     },
@@ -51,6 +65,12 @@ export default {
     },
     mediaProxyData() {
       return _.get(this.settings.settings, [':pleroma', ':media_proxy']) || {}
+    },
+    scriptInvalidation() {
+      return this.settings.description.find(setting => setting.key === 'Pleroma.Web.MediaProxy.Invalidation.Script')
+    },