Commit 53a7680c authored by feld's avatar feld

Merge branch 'develop' into 'fix/mrf-sample-doc'

# Conflicts:
#   docs/configuration/mrf.md
parents 589bffa5 3416948c
*.ex diff=elixir
*.exs diff=elixir
......@@ -101,7 +101,7 @@
%{
key: :versions,
type: {:list, :atom},
description: "List of TLS version to use",
description: "List of TLS versions to use",
suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
}
]
......@@ -534,7 +534,8 @@
%{
key: :description,
type: :string,
description: "The instance's description, can be seen in nodeinfo and /api/v1/instance",
description:
"The instance's description. It can be seen in nodeinfo and `/api/v1/instance`",
suggestions: [
"Very cool instance"
]
......@@ -770,7 +771,7 @@
key: :cleanup_attachments,
type: :boolean,
description: """
"Enable to remove associated attachments when status is removed.
Enable to remove associated attachments when status is removed.
This will not affect duplicates and attachments without status.
Enabling this will increase load to database when deleting statuses on larger instances.
"""
......@@ -838,7 +839,7 @@
%{
key: :healthcheck,
type: :boolean,
description: "If enabled, system data will be shown on /api/pleroma/healthcheck"
description: "If enabled, system data will be shown on `/api/pleroma/healthcheck`"
},
%{
key: :remote_post_retention_days,
......@@ -1296,14 +1297,14 @@
%{
key: :media_removal,
type: {:list, :string},
description: "List of instances to remove medias from",
description: "List of instances to strip media attachments from",
suggestions: ["example.com", "*.example.com"]
},
%{
key: :media_nsfw,
label: "Media NSFW",
type: {:list, :string},
description: "List of instances to put medias as NSFW (sensitive) from",
description: "List of instances to tag all media as NSFW (sensitive) from",
suggestions: ["example.com", "*.example.com"]
},
%{
......@@ -1422,21 +1423,21 @@
key: :reject,
type: [:string, :regex],
description:
"A list of patterns which result in message being rejected, each pattern can be a string or a regular expression.",
"A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
suggestions: ["foo", ~r/foo/iu]
},
%{
key: :federated_timeline_removal,
type: [:string, :regex],
description:
"A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a regular expression.",
"A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
suggestions: ["foo", ~r/foo/iu]
},
%{
key: :replace,
type: [{:tuple, :string, :string}, {:tuple, :regex, :string}],
description:
"A list of tuples containing {pattern, replacement}, pattern can be a string or a regular expression.",
"A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
}
]
......@@ -1451,7 +1452,7 @@
%{
key: :actors,
type: {:list, :string},
description: "A list of actors, for which to drop any posts mentioning",
description: "A list of actors for which any post mentioning them will be dropped.",
suggestions: ["actor1", "actor2"]
}
]
......@@ -1855,9 +1856,8 @@
type: :string,
description:
"A mailto link for the administrative contact." <>
" It's best if this email is not a personal email address, but rather a group email so that if a person leaves an organization," <>
" is unavailable for an extended period, or otherwise can't respond, someone else on the list can.",
suggestions: ["Subject"]
" It's best if this email is not a personal email address, but rather a group email to the instance moderation team.",
suggestions: ["mailto:moderators@pleroma.com"]
},
%{
key: :public_key,
......@@ -1924,7 +1924,7 @@
key: :admin_token,
type: :string,
description: "Token",
suggestions: ["some_random_token"]
suggestions: ["We recommend a secure random string or UUID"]
}
]
},
......@@ -1986,6 +1986,7 @@
"Background jobs queues (keys: queues, values: max numbers of concurrent jobs)",
suggestions: [
activity_expiration: 10,
attachments_cleanup: 5,
background: 5,
federator_incoming: 50,
federator_outgoing: 50,
......@@ -2001,6 +2002,12 @@
description: "Activity expiration queue",
suggestions: [10]
},
%{
key: :attachments_cleanup,
type: :integer,
description: "Attachment deletion queue",
suggestions: [5]
},
%{
key: :background,
type: :integer,
......@@ -2099,7 +2106,7 @@
%{
key: :enabled,
type: :boolean,
description: "Enables/disables RichMedia."
description: "Enables RichMedia parsing of URLs."
},
%{
key: :ignore_hosts,
......@@ -2145,8 +2152,7 @@
%{
key: :enabled,
type: :boolean,
description:
"If enabled, when a new user is federated with, fetch some of their latest posts"
description: "Fetch posts when a new user is federated with"
},
%{
key: :pages,
......@@ -2165,13 +2171,13 @@
%{
key: :class,
type: [:string, false],
description: "Specify the class to be added to the generated link. `False` to clear",
description: "Specify the class to be added to the generated link. Disable to clear",
suggestions: ["auto-linker", false]
},
%{
key: :rel,
type: [:string, false],
description: "Override the rel attribute. `False` to clear",
description: "Override the rel attribute. Disable to clear",
suggestions: ["ugc", "noopener noreferrer", false]
},
%{
......@@ -2281,7 +2287,7 @@
key: :ssl,
label: "SSL",
type: :boolean,
description: "`True` to use SSL, usually implies the port 636"
description: "Enable to use SSL, usually implies the port 636"
},
%{
key: :sslopts,
......@@ -2308,7 +2314,7 @@
key: :tls,
label: "TLS",
type: :boolean,
description: "`True` to start TLS, usually implies the port 389"
description: "Enable to use STARTTLS, usually implies the port 389"
},
%{
key: :tlsopts,
......@@ -2358,7 +2364,7 @@
description:
"OAuth admin scope requirement toggle. " <>
"If enabled, admin actions explicitly demand admin OAuth scope(s) presence in OAuth token " <>
"(client app must support admin scopes). If `false` and token doesn't have admin scope(s)," <>
"(client app must support admin scopes). If disabled and token doesn't have admin scope(s)," <>
"`is_admin` user flag grants access to admin-specific actions."
},
%{
......@@ -2380,7 +2386,7 @@
key: :oauth_consumer_strategies,
type: {:list, :string},
description:
"The list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable." <>
"The list of enabled OAuth consumer strategies. By default it's set by OAUTH_CONSUMER_STRATEGIES environment variable." <>
" Each entry in this space-delimited string should be of format \"strategy\" or \"strategy:dependency\"" <>
" (e.g. twitter or keycloak:ueberauth_keycloak_strategy in case dependency is named differently than ueberauth_<strategy>).",
suggestions: ["twitter", "keycloak:ueberauth_keycloak_strategy"]
......@@ -2517,7 +2523,7 @@
%{
key: :clean_expired_tokens,
type: :boolean,
description: "Enable a background job to clean expired oauth tokens. Default: `false`."
description: "Enable a background job to clean expired oauth tokens. Default: disabled."
}
]
},
......@@ -2577,7 +2583,7 @@
%{
key: :rum_enabled,
type: :boolean,
description: "If RUM indexes should be used. Default: `false`"
description: "If RUM indexes should be used. Default: disabled"
}
]
},
......@@ -2963,7 +2969,7 @@
%{
key: :enabled,
type: :boolean,
description: "Enable/disable the plug. Default: `false`."
description: "Enable/disable the plug. Default: disabled."
},
%{
key: :headers,
......@@ -3017,7 +3023,7 @@
%{
key: :enabled,
type: :boolean,
description: "Enables the rendering of static HTML. Defaults to `false`."
description: "Enables the rendering of static HTML. Default: disabled."
}
]
},
......@@ -3093,7 +3099,7 @@
key: :configurable_from_database,
type: :boolean,
description:
"Allow transferring configuration to DB with the subsequent customization from Admin api. Defaults to `false`"
"Allow transferring configuration to DB with the subsequent customization from Admin api. Default: disabled"
}
]
}
......
......@@ -682,6 +682,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
### Get list of merged default settings with saved in database.
*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
**Only works when configuration from database is enabled.**
- Params:
......@@ -692,20 +694,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
```json
{
configs: [
"configs": [
{
"group": ":pleroma",
"key": "Pleroma.Upload",
"value": []
}
]
],
"need_reboot": true
}
```
need_reboot - *optional*, if were changed reboot time settings.
## `POST /api/pleroma/admin/config`
### Update config settings
*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
**Only works when configuration from database is enabled.**
Some modifications are necessary to save the config settings correctly:
......@@ -793,7 +799,7 @@ config :quack,
```
```json
{
configs: [
"configs": [
{"group": ":quack", "key": ":level", "value": ":debug"},
{"group": ":quack", "key": ":meta", "value": [":all"]},
...
......@@ -804,7 +810,7 @@ config :quack,
```json
{
configs: [
"configs": [
{
"group": ":pleroma",
"key": "Pleroma.Upload",
......@@ -836,15 +842,17 @@ config :quack,
- 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
```json
{
configs: [
"configs": [
{
"group": ":pleroma",
"key": "Pleroma.Upload",
"value": [...]
}
]
],
"need_reboot": true
}
```
need_reboot - *optional*, if were changed reboot time settings.
## ` GET /api/pleroma/admin/config/descriptions`
......
# Message Rewrite Facility
The Message Rewrite Facility (MRF) is a subsystem that is implemented as a series of hooks that allows the administrator to rewrite or discard messages.
Possible uses include:
......@@ -10,7 +11,8 @@ Possible uses include:
* removing media from messages
* sending only public messages to a specific instance
The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
It is possible to use multiple, active MRF policies at the same time.
## Quarantine Instances
......@@ -18,7 +20,8 @@ It is possible to use multiple, active MRF policies at the same time.
You have the ability to prevent from private / followers-only messages from federating with specific instances. Which means they will only get the public or unlisted messages from your instance.
If, for example, you're using `MIX_ENV=prod` aka using production mode, you would open your configuration file located in `config/prod.secret.exs` and edit or add the option under your `:instance` config object. Then you would specify the instance within quotes.
```
```elixir
config :pleroma, :instance,
[...]
quarantined_instances: ["instance.example", "other.example"]
......@@ -28,15 +31,15 @@ config :pleroma, :instance,
`SimplePolicy` is capable of handling most common admin tasks.
To use `SimplePolicy`, you must enable it. Do so by adding the following to your `:instance` config object, so that it looks like this:
To use `SimplePolicy`, you must enable it. Do so by adding the following to your `:instance` config object, so that it looks like this:
```
```elixir
config :pleroma, :instance,
[...]
rewrite_policy: Pleroma.Web.ActivityPub.MRF.SimplePolicy
```
Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object. These groups are:
Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object. These groups are:
* `media_removal`: Servers in this group will have media stripped from incoming messages.
* `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
......@@ -50,7 +53,7 @@ Servers should be configured as lists.
This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
```
```elixir
config :pleroma, :instance,
rewrite_policy: [Pleroma.Web.ActivityPub.MRF.SimplePolicy]
......@@ -60,16 +63,15 @@ config :pleroma, :mrf_simple,
reject: ["spam.com"],
federated_timeline_removal: ["spam.university"],
report_removal: ["whiny.whiner"]
```
### Use with Care
The effects of MRF policies can be very drastic. It is important to use this functionality carefully. Always try to talk to an admin before writing an MRF policy concerning their instance.
The effects of MRF policies can be very drastic. It is important to use this functionality carefully. Always try to talk to an admin before writing an MRF policy concerning their instance.
## Writing your own MRF Policy
As discussed above, the MRF system is a modular system that supports pluggable policies. This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `rewrite_policy` config setting.
As discussed above, the MRF system is a modular system that supports pluggable policies. This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `rewrite_policy` config setting.
For example, here is a sample policy module which rewrites all messages to "new message content":
......@@ -118,7 +120,7 @@ end
If you save this file as `lib/pleroma/web/activity_pub/mrf/rewrite_policy.ex`, it will be included when you next rebuild Pleroma. You can enable it in the configuration like so:
```
```elixir
config :pleroma, :instance,
rewrite_policy: [
Pleroma.Web.ActivityPub.MRF.SimplePolicy,
......@@ -126,4 +128,4 @@ config :pleroma, :instance,
]
```
Please note that the Pleroma developers consider custom MRF policy modules to fall under the purview of the AGPL. As such, you are obligated to release the sources to your custom MRF policy modules upon request.
Please note that the Pleroma developers consider custom MRF policy modules to fall under the purview of the AGPL. As such, you are obligated to release the sources to your custom MRF policy modules upon request.
......@@ -146,9 +146,7 @@ defp group_and_subkey_need_reboot?(group, key, value) do
defp update_env(group, key, nil), do: Application.delete_env(group, key)
defp update_env(group, key, value), do: Application.put_env(group, key, value)
defp restart(_, :pleroma, :test), do: Logger.warn("pleroma restarted")
defp restart(_, :pleroma, _), do: send(Restarter.Pleroma, :after_boot)
defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
defp restart(started_applications, app, _) do
with {^app, _, _} <- List.keyfind(started_applications, app, 0),
......
......@@ -108,6 +108,7 @@ def extract_first_external_url(object, content) do
Cachex.fetch!(:scrubber_cache, key, fn _key ->
result =
content
|> Floki.parse_fragment!()
|> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]")
|> Floki.attribute("a", "href")
|> Enum.at(0)
......
......@@ -17,6 +17,7 @@ defp old_user?(%User{} = u) do
# does the post contain links?
defp contains_links?(%{"content" => content} = _object) do
content
|> Floki.parse_fragment!()
|> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"],a.zrl")
|> Floki.attribute("a", "href")
|> length() > 0
......
......@@ -5,12 +5,11 @@
defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.Web.ActivityPub.MRF
require Pleroma.Constants
@moduledoc "Filter activities depending on their age"
@behaviour MRF
@behaviour Pleroma.Web.ActivityPub.MRF
defp check_date(%{"published" => published} = message) do
with %DateTime{} = now <- DateTime.utc_now(),
......
......@@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.MRF
@moduledoc "Filter activities depending on their origin instance"
@behaviour MRF
@behaviour Pleroma.Web.ActivityPub.MRF
require Pleroma.Constants
......
......@@ -8,7 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
require Logger
@behaviour MRF
@behaviour Pleroma.Web.ActivityPub.MRF
defp lookup_subchain(actor) do
with matches <- Config.get([:mrf_subchain, :match_actor]),
......
......@@ -8,6 +8,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
import Pleroma.Web.ControllerHelper, only: [json_response: 3]
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.ConfigDB
alias Pleroma.ModerationLog
alias Pleroma.Plugs.OAuthScopesPlug
......@@ -570,8 +571,8 @@ def relay_unfollow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target})
@doc "Sends registration invite via email"
def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) do
with true <-
Pleroma.Config.get([:instance, :invites_enabled]) &&
!Pleroma.Config.get([:instance, :registrations_open]),
Config.get([:instance, :invites_enabled]) &&
!Config.get([:instance, :registrations_open]),
{:ok, invite_token} <- UserInviteToken.create_invite(),
email <-
Pleroma.Emails.UserEmail.user_invitation_email(
......@@ -808,7 +809,7 @@ def config_show(conn, _params) do
configs = ConfigDB.get_all_as_keyword()
merged =
Pleroma.Config.Holder.config()
Config.Holder.config()
|> ConfigDB.merge(configs)
|> Enum.map(fn {group, values} ->
Enum.map(values, fn {key, value} ->
......@@ -838,7 +839,16 @@ def config_show(conn, _params) do
end)
|> List.flatten()
json(conn, %{configs: merged})
response = %{configs: merged}
response =
if Restarter.Pleroma.need_reboot?() do
Map.put(response, :need_reboot, true)
else
response
end
json(conn, response)
end
end
......@@ -863,20 +873,26 @@ def config_update(conn, %{"configs" => configs}) do
Ecto.get_meta(config, :state) == :deleted
end)
Pleroma.Config.TransferTask.load_and_update_env(deleted, false)
Config.TransferTask.load_and_update_env(deleted, false)
need_reboot? =
Enum.any?(updated, fn config ->
group = ConfigDB.from_string(config.group)
key = ConfigDB.from_string(config.key)
value = ConfigDB.from_binary(config.value)
Pleroma.Config.TransferTask.pleroma_need_restart?(group, key, value)
end)
Restarter.Pleroma.need_reboot?() ||
Enum.any?(updated, fn config ->
group = ConfigDB.from_string(config.group)
key = ConfigDB.from_string(config.key)
value = ConfigDB.from_binary(config.value)
Config.TransferTask.pleroma_need_restart?(group, key, value)
end)
response = %{configs: updated}
response =
if need_reboot?, do: Map.put(response, :need_reboot, need_reboot?), else: response
if need_reboot? do
Restarter.Pleroma.need_reboot()
Map.put(response, :need_reboot, need_reboot?)
else
response
end
conn
|> put_view(ConfigView)
......@@ -886,18 +902,14 @@ def config_update(conn, %{"configs" => configs}) do
def restart(conn, _params) do
with :ok <- configurable_from_database(conn) do
if Pleroma.Config.get(:env) == :test do
Logger.warn("pleroma restarted")
else
send(Restarter.Pleroma, {:restart, 50})
end
Restarter.Pleroma.restart(Config.get(:env), 50)
json(conn, %{})
end
end
defp configurable_from_database(conn) do
if Pleroma.Config.get(:configurable_from_database) do
if Config.get(:configurable_from_database) do
:ok
else
errors(
......
......@@ -175,9 +175,11 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
expires_at =
with true <- client_posted_this_activity,
expiration when not is_nil(expiration) <-
%ActivityExpiration{scheduled_at: scheduled_at} <-
ActivityExpiration.get_by_activity_id(activity.id) do
expiration.scheduled_at
scheduled_at
else
_ -> nil
end
thread_muted? =
......@@ -321,11 +323,9 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
nil
end
site_name = rich_media[:site_name] || page_url_data.host
%{
type: "link",
provider_name: site_name,
provider_name: page_url_data.host,
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url |> MediaProxy.url(),
......
......@@ -8,8 +8,10 @@ defmodule Pleroma.Web.Metadata.Providers.RelMe do
@impl Provider
def build_tags(%{user: user}) do
(Floki.attribute(user.bio, "link[rel~=me]", "href") ++
Floki.attribute(user.bio, "a[rel~=me]", "href"))
bio_tree = Floki.parse_fragment!(user.bio)
(Floki.attribute(bio_tree, "link[rel~=me]", "href") ++
Floki.attribute(bio_tree, "a[rel~=me]", "href"))
|> Enum.map(fn link ->
{:link, [rel: "me", href: link], []}
end)
......
......@@ -27,9 +27,10 @@ def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
Pleroma.HTTP.get(url, [], adapter: @hackney_options),
{:ok, html_tree} <- Floki.parse_document(html),
data <-
Floki.attribute(html, "link[rel~=me]", "href") ++
Floki.attribute(html, "a[rel~=me]", "href") do
Floki.attribute(html_tree, "link[rel~=me]", "href") ++
Floki.attribute(html_tree, "a[rel~=me]", "href") do
{:ok, data}
end
rescue
......
......@@ -81,18 +81,18 @@ defp parse_url(url) do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
html
|> parse_html
|> parse_html()
|> maybe_parse()
|> Map.put(:url, url)
|> clean_parsed_data()
|> check_parsed_data()
rescue
e ->
{:error, "Parsing error: #{inspect(e)}"}
{:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
end
end
defp parse_html(html), do: Floki.parse(html)