Commit 4e81b4b1 authored by lain's avatar lain

Merge branch 'relations-preloading-for-statuses-rendering' into 'develop'

Performance improvements (timeline / statuses / notifications / accounts rendering)

See merge request !2323
parents af820f8c dfbc05d4
Pipeline #23973 passed with stages
in 45 minutes and 23 seconds
......@@ -35,6 +35,13 @@ def by_author(query \\ Activity, %User{ap_id: ap_id}) do
from(a in query, where: a.actor == ^ap_id)
end
def find_by_object_ap_id(activities, object_ap_id) do
Enum.find(
activities,
&(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]])
)
end
@spec by_object_id(query, String.t() | [String.t()]) :: query
def by_object_id(query \\ Activity, object_id)
......
......@@ -129,21 +129,18 @@ def for_user(user, params \\ %{}) do
end
def restrict_recipients(query, user, %{"recipients" => user_ids}) do
user_ids =
user_binary_ids =
[user.id | user_ids]
|> Enum.uniq()
|> Enum.reduce([], fn user_id, acc ->
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
[user_id | acc]
end)
|> User.binary_id()
conversation_subquery =
__MODULE__
|> group_by([p], p.conversation_id)
|> having(
[p],
count(p.user_id) == ^length(user_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
count(p.user_id) == ^length(user_binary_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
)
|> select([p], %{id: p.conversation_id})
......
......@@ -129,4 +129,32 @@ def move_following(origin, target) do
move_following(origin, target)
end
end
def all_between_user_sets(
source_users,
target_users
)
when is_list(source_users) and is_list(target_users) do
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
__MODULE__
|> where(
fragment(
"(follower_id = ANY(?) AND following_id = ANY(?)) OR \
(follower_id = ANY(?) AND following_id = ANY(?))",
^source_user_ids,
^target_user_ids,
^target_user_ids,
^source_user_ids
)
)
|> Repo.all()
end
def find(following_relationships, follower, following) do
Enum.find(following_relationships, fn
fr -> fr.follower_id == follower.id and fr.following_id == following.id
end)
end
end
......@@ -25,10 +25,10 @@ def changeset(mute, params \\ %{}) do
end
def query(user_id, context) do
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
user_binary_id = User.binary_id(user_id)
ThreadMute
|> where(user_id: ^user_id)
|> where(user_id: ^user_binary_id)
|> where(context: ^context)
end
......@@ -68,8 +68,8 @@ def remove_mute(user_id, context) do
|> Repo.delete_all()
end
def check_muted(user_id, context) do
def exists?(user_id, context) do
query(user_id, context)
|> Repo.all()
|> Repo.exists?()
end
end
......@@ -226,6 +226,24 @@ def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \
end
end
@doc """
Dumps Flake Id to SQL-compatible format (16-byte UUID).
E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>>
"""
def binary_id(source_id) when is_binary(source_id) do
with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do
dumped_id
else
_ -> source_id
end
end
def binary_id(source_ids) when is_list(source_ids) do
Enum.map(source_ids, &binary_id/1)
end
def binary_id(%User{} = user), do: binary_id(user.id)
@doc "Returns status account"
@spec account_status(User.t()) :: account_status()
def account_status(%User{deactivated: true}), do: :deactivated
......@@ -753,7 +771,14 @@ def unfollow(%User{} = follower, %User{} = followed) do
def get_follow_state(%User{} = follower, %User{} = following) do
following_relationship = FollowingRelationship.get(follower, following)
get_follow_state(follower, following, following_relationship)
end
def get_follow_state(
%User{} = follower,
%User{} = following,
following_relationship
) do
case {following_relationship, following.local} do
{nil, false} ->
case Utils.fetch_latest_follow(follower, following) do
......@@ -1747,8 +1772,12 @@ def all_superusers do
|> Repo.all()
end
def muting_reblogs?(%User{} = user, %User{} = target) do
UserRelationship.reblog_mute_exists?(user, target)
end
def showing_reblogs?(%User{} = user, %User{} = target) do
not UserRelationship.reblog_mute_exists?(user, target)
not muting_reblogs?(user, target)
end
@doc """
......
......@@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do
import Ecto.Changeset
import Ecto.Query
alias Pleroma.FollowingRelationship
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.UserRelationship
......@@ -37,6 +38,10 @@ def unquote(:"#{relationship_type}_exists?")(source, target),
do: exists?(unquote(relationship_type), source, target)
end
def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__()
def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
user_relationship
|> cast(params, [:relationship_type, :source_id, :target_id])
......@@ -75,6 +80,73 @@ def delete(relationship_type, %User{} = source, %User{} = target) do
end
end
def dictionary(
source_users,
target_users,
source_to_target_rel_types \\ nil,
target_to_source_rel_types \\ nil
)
when is_list(source_users) and is_list(target_users) do
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end
source_to_target_rel_types =
Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
target_to_source_rel_types =
Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
__MODULE__
|> where(
fragment(
"(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \
(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))",
^source_user_ids,
^target_user_ids,
^source_to_target_rel_types,
^target_user_ids,
^source_user_ids,
^target_to_source_rel_types
)
)
|> select([ur], [ur.relationship_type, ur.source_id, ur.target_id])
|> Repo.all()
end
def exists?(dictionary, rel_type, source, target, func) do
cond do
is_nil(source) or is_nil(target) ->
false
dictionary ->
[rel_type, source.id, target.id] in dictionary
true ->
func.(source, target)
end
end
@doc ":relationships option for StatusView / AccountView / NotificationView"
def view_relationships_option(nil = _reading_user, _actors) do
%{user_relationships: [], following_relationships: []}
end
def view_relationships_option(%User{} = reading_user, actors) do
user_relationships =
UserRelationship.dictionary(
[reading_user],
actors,
[:block, :mute, :notification_mute, :reblog_mute],
[:block, :inverse_subscription]
)
following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors)
%{user_relationships: user_relationships, following_relationships: following_relationships}
end
defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do
changeset
|> validate_change(:target_id, fn _, target_id ->
......
......@@ -358,7 +358,7 @@ def remove_mute(user, activity) do
def thread_muted?(%{id: nil} = _user, _activity), do: false
def thread_muted?(user, activity) do
ThreadMute.check_muted(user.id, activity.data["context"]) != []
ThreadMute.exists?(user.id, activity.data["context"])
end
def report(user, %{"account_id" => account_id} = data) do
......
......@@ -5,12 +5,28 @@
defmodule Pleroma.Web.MastodonAPI.AccountView do
use Pleroma.Web, :view
alias Pleroma.FollowingRelationship
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MediaProxy
def render("index.json", %{users: users} = opts) do
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(opts[:for]) ->
UserRelationship.view_relationships_option(nil, [])
true ->
UserRelationship.view_relationships_option(opts[:for], users)
end
opts = Map.put(opts, :relationships, relationships_opt)
users
|> render_many(AccountView, "show.json", opts)
|> Enum.filter(&Enum.any?/1)
......@@ -35,27 +51,107 @@ def render("relationship.json", %{user: nil, target: _target}) do
%{}
end
def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do
follow_state = User.get_follow_state(user, target)
def render(
"relationship.json",
%{user: %User{} = reading_user, target: %User{} = target} = opts
) do
user_relationships = get_in(opts, [:relationships, :user_relationships])
following_relationships = get_in(opts, [:relationships, :following_relationships])
follow_state =
if following_relationships do
user_to_target_following_relation =
FollowingRelationship.find(following_relationships, reading_user, target)
User.get_follow_state(reading_user, target, user_to_target_following_relation)
else
User.get_follow_state(reading_user, target)
end
followed_by =
if following_relationships do
case FollowingRelationship.find(following_relationships, target, reading_user) do
%{state: "accept"} -> true
_ -> false
end
else
User.following?(target, reading_user)
end
# NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags
%{
id: to_string(target.id),
following: follow_state == "accept",
followed_by: User.following?(target, user),
blocking: User.blocks_user?(user, target),
blocked_by: User.blocks_user?(target, user),
muting: User.mutes?(user, target),
muting_notifications: User.muted_notifications?(user, target),
subscribing: User.subscribed_to?(user, target),
followed_by: followed_by,
blocking:
UserRelationship.exists?(
user_relationships,
:block,
reading_user,
target,
&User.blocks_user?(&1, &2)
),
blocked_by:
UserRelationship.exists?(
user_relationships,
:block,
target,
reading_user,
&User.blocks_user?(&1, &2)
),
muting:
UserRelationship.exists?(
user_relationships,
:mute,
reading_user,
target,
&User.mutes?(&1, &2)
),
muting_notifications:
UserRelationship.exists?(
user_relationships,
:notification_mute,
reading_user,
target,
&User.muted_notifications?(&1, &2)
),
subscribing:
UserRelationship.exists?(
user_relationships,
:inverse_subscription,
target,
reading_user,
&User.subscribed_to?(&2, &1)
),
requested: follow_state == "pending",
domain_blocking: User.blocks_domain?(user, target),
showing_reblogs: User.showing_reblogs?(user, target),
domain_blocking: User.blocks_domain?(reading_user, target),
showing_reblogs:
not UserRelationship.exists?(
user_relationships,
:reblog_mute,
reading_user,
target,
&User.muting_reblogs?(&1, &2)
),
endorsed: false
}
end
def render("relationships.json", %{user: user, targets: targets}) do
render_many(targets, AccountView, "relationship.json", user: user, as: :target)
def render("relationships.json", %{user: user, targets: targets} = opts) do
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(opts[:for]) ->
UserRelationship.view_relationships_option(nil, [])
true ->
UserRelationship.view_relationships_option(user, targets)
end
render_opts = %{as: :target, user: user, relationships: relationships_opt}
render_many(targets, AccountView, "relationship.json", render_opts)
end
defp do_render("show.json", %{user: user} = opts) do
......@@ -93,7 +189,12 @@ defp do_render("show.json", %{user: user} = opts) do
}
end)
relationship = render("relationship.json", %{user: opts[:for], target: user})
relationship =
render("relationship.json", %{
user: opts[:for],
target: user,
relationships: opts[:relationships]
})
%{
id: to_string(user.id),
......
......@@ -8,24 +8,86 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
alias Pleroma.Activity
alias Pleroma.Notification
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MastodonAPI.NotificationView
alias Pleroma.Web.MastodonAPI.StatusView
def render("index.json", %{notifications: notifications, for: user}) do
safe_render_many(notifications, NotificationView, "show.json", %{for: user})
def render("index.json", %{notifications: notifications, for: reading_user} = opts) do
activities = Enum.map(notifications, & &1.activity)
parent_activities =
activities
|> Enum.filter(
&(Activity.mastodon_notification_type(&1) in [
"favourite",
"reblog",
"pleroma:emoji_reaction"
])
)
|> Enum.map(& &1.data["object"])
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object(:left)
|> Pleroma.Repo.all()
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(opts[:for]) ->
UserRelationship.view_relationships_option(nil, [])
true ->
move_activities_targets =
activities
|> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move"))
|> Enum.map(&User.get_cached_by_ap_id(&1.data["target"]))
actors =
activities
|> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end)
|> Enum.filter(& &1)
|> Kernel.++(move_activities_targets)
UserRelationship.view_relationships_option(reading_user, actors)
end
opts = %{
for: reading_user,
parent_activities: parent_activities,
relationships: relationships_opt
}
safe_render_many(notifications, NotificationView, "show.json", opts)
end
def render("show.json", %{
notification: %Notification{activity: activity} = notification,
for: user
}) do
def render(
"show.json",
%{
notification: %Notification{activity: activity} = notification,
for: reading_user
} = opts
) do
actor = User.get_cached_by_ap_id(activity.data["actor"])
parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
parent_activity_fn = fn ->
if opts[:parent_activities] do
Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"])
else
Activity.get_create_by_object_ap_id(activity.data["object"])
end
end
mastodon_type = Activity.mastodon_notification_type(activity)
with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do
with %{id: _} = account <-
AccountView.render("show.json", %{
user: actor,
for: reading_user,
relationships: opts[:relationships]
}) do
response = %{
id: to_string(notification.id),
type: mastodon_type,
......@@ -36,24 +98,28 @@ def render("show.json", %{
}
}
render_opts = %{relationships: opts[:relationships]}
case mastodon_type do
"mention" ->
put_status(response, activity, user)
put_status(response, activity, reading_user, render_opts)
"favourite" ->
put_status(response, parent_activity, user)
put_status(response, parent_activity_fn.(), reading_user, render_opts)
"reblog" ->
put_status(response, parent_activity, user)
put_status(response, parent_activity_fn.(), reading_user, render_opts)
"move" ->
put_target(response, activity, user)
put_target(response, activity, reading_user, render_opts)
"follow" ->
response
"pleroma:emoji_reaction" ->
put_status(response, parent_activity, user) |> put_emoji(activity)
response
|> put_status(parent_activity_fn.(), reading_user, render_opts)
|> put_emoji(activity)
_ ->
nil
......@@ -64,16 +130,21 @@ def render("show.json", %{
end
defp put_emoji(response, activity) do
response
|> Map.put(:emoji, activity.data["content"])
Map.put(response, :emoji, activity.data["content"])
end
defp put_status(response, activity, user) do
Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user}))
defp put_status(response, activity, reading_user, opts) do
status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user})
status_render = StatusView.render("show.json", status_render_opts)
Map.put(response, :status, status_render)
end
defp put_target(response, activity, user) do
target = User.get_cached_by_ap_id(activity.data["target"])
Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user}))
defp put_target(response, activity, reading_user, opts) do
target_user = User.get_cached_by_ap_id(activity.data["target"])
target_render_opts = Map.merge(opts, %{user: target_user, for: reading_user})
target_render = AccountView.render("show.json", target_render_opts)
Map.put(response, :target, target_render)
end
end
......@@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
......@@ -71,10 +72,41 @@ defp reblogged?(activity, user) do
end
def render("index.json", opts) do
replied_to_activities = get_replied_to_activities(opts.activities)
opts = Map.put(opts, :replied_to_activities, replied_to_activities)
# To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
activities = Enum.filter(opts.activities, & &1)
replied_to_activities = get_replied_to_activities(activities)
safe_render_many(opts.activities, StatusView, "show.json", opts)
parent_activities =
activities
|> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"]))
|> Enum.map(&Object.normalize(&1).data["id"])
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object(:left)
|> Activity.with_preloaded_bookmark(opts[:for])
|> Activity.with_set_thread_muted_field(opts[:for])
|> Repo.all()
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(opts[:for]) ->
UserRelationship.view_relationships_option(nil, [])
true ->
actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"]))
UserRelationship.view_relationships_option(opts[:for], actors)
end
opts =
opts
|> Map.put(:replied_to_activities, replied_to_activities)
|> Map.put(:parent_activities, parent_activities)
|> Map.put(:relationships, relationships_opt)
safe_render_many(activities, StatusView, "show.json", opts)
end
def render(
......@@ -85,17 +117,25 @@ def render(
created_at = Utils.to_masto_date(activity.data["published"])
activity_object = Object.normalize(activity)
reblogged_activity =
Activity.create_by_object_ap_id(activity_object.data["id"])
|> Activity.with_preloaded_bookmark(opts[:for])
|> Activity.with_set_thread_muted_field(opts[:for])
|> Repo.one()
reblogged_parent_activity =
if opts[:parent_activities] do
Activity.Queries.find_by_object_ap_id(
opts[:parent_activities],
activity_object.data["id"]
)
else
Activity.create_by_object_ap_id(activity_object.data["id"])
|> Activity.with_preloaded_bookmark(opts[:for])
|> Activity.with_set_thread_muted_field(opts[:for])
|> Repo.one()
end
reblogged = render("show.json", Map.put(opts, :activity, reblogged_activity))
reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity)
reblogged = render("show.json", reblog_rendering_opts)
favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || [])
bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil
bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil
mentions =
activity.recipients
......@@ -107,7 +147,12 @@ def render(
id: to_string(activity.id),
uri: activity_object.data["id"],
url: activity_object.data["id"],
account: AccountView.render("show.json", %{user: user, for: opts[:for]}),
account:
AccountView.render("show.json", %{
user: user,
for: opts[:for],
relationships: opts[:relationships]
}),
in_reply_to_id: nil,
in_reply_to_account_id: nil,
reblog: reblogged,
......@@ -116,7 +161,7 @@ def render(
reblogs_count: 0,
replies_count: 0,
favourites_count: 0,
reblogged: reblogged?(reblogged_activity, opts[:for]),
reblogged: reblogged?(reblogged_parent_activity, opts[:for]),
favourited: present?(favorited),
bookmarked: present?(bookmarked),
muted: false,
......@@ -183,9 +228,10 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
end
thread_muted? =
case activity.thread_muted? do