Verified Commit 58e4e3db authored by minibikini's avatar minibikini
Browse files

Merge remote-tracking branch 'origin/develop' into merge-ogp-twitter-parsers

parents 09d31d24 448e93ce
......@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Filtering of push notifications on activities from blocked domains
- Resolving Peertube accounts with Webfinger
- `blob:` urls not being allowed by connect-src CSP
- Mastodon API: fix `GET /api/v1/notifications` not returning the full result set
## [Unreleased (patch)]
......
......@@ -1623,14 +1623,12 @@
# %{
# group: :pleroma,
# key: :mrf_user_allowlist,
# type: :group,
# type: :map,
# description:
# "The keys in this section are the domain names that the policy should apply to." <>
# " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
# children: [
# ["example.org": ["https://example.org/users/admin"]],
# suggestions: [
# ["example.org": ["https://example.org/users/admin"]]
# %{"example.org" => ["https://example.org/users/admin"]}
# ]
# ]
# },
......
......@@ -138,8 +138,9 @@ their ActivityPub ID.
An example:
```elixir
config :pleroma, :mrf_user_allowlist,
"example.org": ["https://example.org/users/admin"]
config :pleroma, :mrf_user_allowlist, %{
"example.org" => ["https://example.org/users/admin"]
}
```
#### :mrf_object_age
......
......@@ -37,18 +37,17 @@ server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_session_timeout 5m;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
ssl_trusted_certificate /etc/letsencrypt/live/example.tld/chain.pem;
ssl_certificate /etc/letsencrypt/live/example.tld/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.tld/privkey.pem;
# Add TLSv1.0 to support older devices
ssl_protocols TLSv1.2;
# Uncomment line below if you want to support older devices (Before Android 4.4.2, IE 8, etc.)
# ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
ssl_prefer_server_ciphers on;
ssl_prefer_server_ciphers off;
# In case of an old server with an OpenSSL version of 1.0.2 or below,
# leave only prime256v1 or comment out the following line.
ssl_ecdh_curve X25519:prime256v1:secp384r1:secp521r1;
......
......@@ -31,6 +31,10 @@ defmodule Pleroma.Activity do
field(:recipients, {:array, :string}, default: [])
field(:thread_muted?, :boolean, virtual: true)
# A field that can be used if you need to join some kind of other
# id to order / paginate this field by
field(:pagination_id, :string, virtual: true)
# This is a fake relation,
# do not use outside of with_preloaded_user_actor/with_joined_user_actor
has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id)
......
......@@ -4,9 +4,10 @@
defmodule Pleroma.Config.DeprecationWarnings do
require Logger
alias Pleroma.Config
def check_hellthread_threshold do
if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
if Config.get([:mrf_hellthread, :threshold]) do
Logger.warn("""
!!!DEPRECATION WARNING!!!
You are using the old configuration mechanism for the hellthread filter. Please check config.md.
......@@ -14,7 +15,29 @@ def check_hellthread_threshold do
end
end
def mrf_user_allowlist do
config = Config.get(:mrf_user_allowlist)
if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
rewritten =
Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
Map.put(acc, to_string(k), v)
end)
Config.put(:mrf_user_allowlist, rewritten)
Logger.error("""
!!!DEPRECATION WARNING!!!
As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
""")
end
end
def warn do
check_hellthread_threshold()
mrf_user_allowlist()
end
end
......@@ -166,8 +166,16 @@ defp exclude_visibility(query, %{exclude_visibilities: visibility})
query
|> join(:left, [n, a], mutated_activity in Pleroma.Activity,
on:
fragment("?->>'context'", a.data) ==
fragment("?->>'context'", mutated_activity.data) and
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
a.data,
a.data
) ==
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
mutated_activity.data,
mutated_activity.data
) and
fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
fragment("?->>'type'", mutated_activity.data) == "Create",
as: :mutated_activity
......@@ -541,6 +549,7 @@ def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
def skip?(%Activity{} = activity, %User{} = user) do
[
:self,
:invisible,
:followers,
:follows,
:non_followers,
......@@ -557,6 +566,12 @@ def skip?(:self, %Activity{} = activity, %User{} = user) do
activity.data["actor"] == user.ap_id
end
def skip?(:invisible, %Activity{} = activity, _) do
actor = activity.data["actor"]
user = User.get_cached_by_ap_id(actor)
User.invisible?(user)
end
def skip?(
:followers,
%Activity{} = activity,
......
......@@ -1488,6 +1488,7 @@ def perform(:delete, %User{} = user) do
end)
delete_user_activities(user)
delete_notifications_from_user_activities(user)
delete_outgoing_pending_follow_requests(user)
......@@ -1576,6 +1577,13 @@ def follow_import(%User{} = follower, followed_identifiers)
})
end
def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
Notification
|> join(:inner, [n], activity in assoc(n, :activity))
|> where([n, a], fragment("? = ?", a.actor, ^ap_id))
|> Repo.delete_all()
end
def delete_user_activities(%User{ap_id: ap_id} = user) do
ap_id
|> Activity.Queries.by_actor()
......
......@@ -32,25 +32,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
require Logger
require Pleroma.Constants
# For Announce activities, we filter the recipients based on following status for any actors
# that match actual users. See issue #164 for more information about why this is necessary.
defp get_recipients(%{"type" => "Announce"} = data) do
to = Map.get(data, "to", [])
cc = Map.get(data, "cc", [])
bcc = Map.get(data, "bcc", [])
actor = User.get_cached_by_ap_id(data["actor"])
recipients =
Enum.filter(Enum.concat([to, cc, bcc]), fn recipient ->
case User.get_cached_by_ap_id(recipient) do
nil -> true
user -> User.following?(user, actor)
end
end)
{recipients, to, cc}
end
defp get_recipients(%{"type" => "Create"} = data) do
to = Map.get(data, "to", [])
cc = Map.get(data, "cc", [])
......@@ -721,6 +702,26 @@ defp user_activities_recipients(%{reading_user: reading_user}) do
end
end
defp restrict_announce_object_actor(_query, %{announce_filtering_user: _, skip_preload: true}) do
raise "Can't use the child object without preloading!"
end
defp restrict_announce_object_actor(query, %{announce_filtering_user: %{ap_id: actor}}) do
from(
[activity, object] in query,
where:
fragment(
"?->>'type' != ? or ?->>'actor' != ?",
activity.data,
"Announce",
object.data,
^actor
)
)
end
defp restrict_announce_object_actor(query, _), do: query
defp restrict_since(query, %{since_id: ""}), do: query
defp restrict_since(query, %{since_id: since_id}) do
......@@ -1144,6 +1145,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
|> restrict_pinned(opts)
|> restrict_muted_reblogs(restrict_muted_reblogs_opts)
|> restrict_instance(opts)
|> restrict_announce_object_actor(opts)
|> Activity.restrict_deactivated_users()
|> exclude_poll_votes(opts)
|> exclude_chat_messages(opts)
......@@ -1170,12 +1172,11 @@ def fetch_favourites(user, params \\ %{}, pagination \\ :keyset) do
|> Activity.Queries.by_type("Like")
|> Activity.with_joined_object()
|> Object.with_joined_activity()
|> select([_like, object, activity], %{activity | object: object})
|> select([like, object, activity], %{activity | object: object, pagination_id: like.id})
|> order_by([like, _, _], desc_nulls_last: like.id)
|> Pagination.fetch_paginated(
Map.merge(params, %{skip_order: true}),
pagination,
:object_activity
pagination
)
end
......
......@@ -24,7 +24,7 @@ def filter(%{"actor" => actor} = object) do
allow_list =
Config.get(
[:mrf_user_allowlist, String.to_atom(actor_info.host)],
[:mrf_user_allowlist, actor_info.host],
[]
)
......
......@@ -333,7 +333,8 @@ def favourites_operation do
%Operation{
tags: ["Statuses"],
summary: "Favourited statuses",
description: "Statuses the user has favourited",
description:
"Statuses the user has favourited. Please note that you have to use the link headers to paginate this. You can not build the query parameters yourself.",
operationId: "StatusController.favourites",
parameters: pagination_params(),
security: [%{"oAuth" => ["read:favourites"]}],
......
......@@ -57,35 +57,36 @@ def add_link_headers(conn, activities, extra_params) do
end
end
@id_keys Pagination.page_keys() -- ["limit", "order"]
defp build_pagination_fields(conn, min_id, max_id, extra_params) do
params =
conn.params
|> Map.drop(Map.keys(conn.path_params))
|> Map.merge(extra_params)
|> Map.drop(@id_keys)
%{
"next" => current_url(conn, Map.put(params, :max_id, max_id)),
"prev" => current_url(conn, Map.put(params, :min_id, min_id)),
"id" => current_url(conn)
}
end
def get_pagination_fields(conn, activities, extra_params \\ %{}) do
case List.last(activities) do
%{id: max_id} ->
params =
conn.params
|> Map.drop(Map.keys(conn.path_params))
|> Map.merge(extra_params)
|> Map.drop(Pagination.page_keys() -- ["limit", "order"])
%{pagination_id: max_id} when not is_nil(max_id) ->
%{pagination_id: min_id} =
activities
|> List.first()
build_pagination_fields(conn, min_id, max_id, extra_params)
min_id =
%{id: max_id} ->
%{id: min_id} =
activities
|> List.first()
|> Map.get(:id)
fields = %{
"next" => current_url(conn, Map.put(params, :max_id, max_id)),
"prev" => current_url(conn, Map.put(params, :min_id, min_id))
}
# Generating an `id` without already present pagination keys would
# need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
# instead of the `q.id > ^min_id` and `q.id < ^max_id`.
# This is because we only have ids present inside of the page, while
# `min_id`, `since_id` and `max_id` requires to know one outside of it.
if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
Map.put(fields, "id", current_url(conn, conn.params))
else
fields
end
build_pagination_fields(conn, min_id, max_id, extra_params)
_ ->
%{}
......
......@@ -48,6 +48,7 @@ def home(%{assigns: %{user: user}} = conn, params) do
|> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user)
|> Map.put(:reply_filtering_user, user)
|> Map.put(:announce_filtering_user, user)
|> Map.put(:user, user)
activities =
......
......@@ -46,6 +46,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op
activities
|> Enum.filter(&(&1.data["type"] == "Move"))
|> Enum.map(&User.get_cached_by_ap_id(&1.data["target"]))
|> Enum.filter(& &1)
actors =
activities
......@@ -84,50 +85,45 @@ def render(
# Note: :relationships contain user mutes (needed for :muted flag in :status)
status_render_opts = %{relationships: opts[:relationships]}
with %{id: _} = account <-
AccountView.render(
"show.json",
%{user: actor, for: reading_user}
) do
response = %{
id: to_string(notification.id),
type: notification.type,
created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
account: account,
pleroma: %{
is_seen: notification.seen
}
account =
AccountView.render(
"show.json",
%{user: actor, for: reading_user}
)
response = %{
id: to_string(notification.id),
type: notification.type,
created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
account: account,
pleroma: %{
is_seen: notification.seen
}
}
case notification.type do
"mention" ->
put_status(response, activity, reading_user, status_render_opts)
case notification.type do
"mention" ->
put_status(response, activity, reading_user, status_render_opts)
"favourite" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
"favourite" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
"reblog" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
"reblog" ->
put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
"move" ->
put_target(response, activity, reading_user, %{})
"move" ->
put_target(response, activity, reading_user, %{})
"pleroma:emoji_reaction" ->
response
|> put_status(parent_activity_fn.(), reading_user, status_render_opts)
|> put_emoji(activity)
"pleroma:emoji_reaction" ->
response
|> put_status(parent_activity_fn.(), reading_user, status_render_opts)
|> put_emoji(activity)
"pleroma:chat_mention" ->
put_chat_message(response, activity, reading_user, status_render_opts)
"pleroma:chat_mention" ->
put_chat_message(response, activity, reading_user, status_render_opts)
type when type in ["follow", "follow_request"] ->
response
_ ->
nil
end
else
_ -> nil
type when type in ["follow", "follow_request"] ->
response
end
end
......
......@@ -377,8 +377,8 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
page_url_data = URI.parse(page_url)
page_url_data =
if rich_media[:url] != nil do
URI.merge(page_url_data, URI.parse(rich_media[:url]))
if is_binary(rich_media["url"]) do
URI.merge(page_url_data, URI.parse(rich_media["url"]))
else
page_url_data
end
......@@ -386,11 +386,9 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
page_url = page_url_data |> to_string
image_url =
if rich_media[:image] != nil do
URI.merge(page_url_data, URI.parse(rich_media[:image]))
if is_binary(rich_media["image"]) do
URI.merge(page_url_data, URI.parse(rich_media["image"]))
|> to_string
else
nil
end
%{
......@@ -399,8 +397,8 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url |> MediaProxy.url(),
title: rich_media[:title] || "",
description: rich_media[:description] || "",
title: rich_media["title"] || "",
description: rich_media["description"] || "",
pleroma: %{
opengraph: rich_media
}
......
......@@ -9,7 +9,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
@spec validate_page_url(any()) :: :ok | :error
@spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
......@@ -18,8 +18,8 @@ defp validate_page_url(page_url) when is_binary(page_url) do
|> parse_uri(page_url)
end
defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
when scheme == "https" and not is_nil(authority) do
defp validate_page_url(%URI{host: host, scheme: "https", authority: authority})
when is_binary(authority) do
cond do
host in Config.get([:rich_media, :ignore_hosts], []) ->
:error
......
......@@ -91,7 +91,7 @@ defp parse_url(url) do
html
|> parse_html()
|> maybe_parse()
|> Map.put(:url, url)
|> Map.put("url", url)
|> clean_parsed_data()
|> check_parsed_data()
rescue
......@@ -111,8 +111,8 @@ defp maybe_parse(html) do
end)
end
defp check_parsed_data(%{title: title} = data)
when is_binary(title) and byte_size(title) > 0 do
defp check_parsed_data(%{"title" => title} = data)
when is_binary(title) and title != "" do
{:ok, data}
end
......@@ -123,11 +123,7 @@ defp check_parsed_data(data) do
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
with {:ok, _} <- Jason.encode(%{key => val}) do
false
else
_ -> true
end
not match?({:ok, _}, Jason.encode(%{key => val}))
end)
|> Map.new()
end
......
......@@ -22,19 +22,19 @@ defp normalize_attributes(html_node, prefix, key_name, value_name) do
{_tag, attributes, _children} = html_node
data =
Enum.into(attributes, %{}, fn {name, value} ->
Map.new(attributes, fn {name, value} ->
{name, String.trim_leading(value, "#{prefix}:")}
end)
%{String.to_atom(data[key_name]) => data[value_name]}
%{data[key_name] => data[value_name]}
end
defp maybe_put_title(%{title: _} = meta, _), do: meta
defp maybe_put_title(%{"title" => _} = meta, _), do: meta
defp maybe_put_title(meta, html) when meta != %{} do
case get_page_title(html) do
"" -> meta
title -> Map.put_new(meta, :title, title)
title -> Map.put_new(meta, "title", title)
end
end
......
......@@ -5,7 +5,7 @@
defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
def parse(html, _data) do
with elements = [_ | _] <- get_discovery_data(html),
{:ok, oembed_url} <- get_oembed_url(elements),
oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
{:ok, oembed_data} <- get_oembed_data(oembed_url) do
oembed_data
else
......@@ -17,19 +17,13 @@ defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
end
defp get_oembed_url(nodes) do
{"link", attributes, _children} = nodes |> hd()
{:ok, Enum.into(attributes, %{})["href"]}
defp get_oembed_url([{"link", attributes, _children} | _]) do
Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
end
defp get_oembed_data(url) do
{:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
{:ok, data} = Jason.decode(json)
data = data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end)
{:ok, data}
with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do
Jason.decode(json)
end
end
end
defmodule Pleroma.Repo.Migrations.DeleteNotificationsFromInvisibleUsers do
use Ecto.Migration