Commit 09f10b59 authored by feld's avatar feld

Merge branch 'mfc-timeline-perf' into mfc-timeline-perf-with-virt-scroll

parents 4dd188ff 8b61c7a9
......@@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise).
- Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try.
- Mastodon API: Limit timeline requests to 3 per timeline per 500ms per user/ip by default.
- Admin API: `PATCH /api/pleroma/admin/users/:nickname/credentials` and `GET /api/pleroma/admin/users/:nickname/credentials`
</details>
### Added
......
......@@ -22,9 +22,10 @@ def generate_like_activities(user, posts) do
def generate_users(opts) do
IO.puts("Starting generating #{opts[:users_max]} users...")
{time, _} = :timer.tc(fn -> do_generate_users(opts) end)
{time, users} = :timer.tc(fn -> do_generate_users(opts) end)
IO.puts("Inserting users take #{to_sec(time)} sec.\n")
IO.puts("Inserting users took #{to_sec(time)} sec.\n")
users
end
defp do_generate_users(opts) do
......
defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator
alias Pleroma.Web.CommonAPI
def run(_args) do
Mix.Pleroma.start_pleroma()
# Cleaning tables
clean_tables()
[{:ok, user} | users] = Generator.generate_users(users_max: 1000)
# Let the user make 100 posts
1..100
|> Enum.each(fn i -> CommonAPI.post(user, %{"status" => to_string(i)}) end)
# Let 10 random users post
posts =
users
|> Enum.take_random(10)
|> Enum.map(fn {:ok, random_user} ->
{:ok, activity} = CommonAPI.post(random_user, %{"status" => "."})
activity
end)
# let our user repeat them
posts
|> Enum.each(fn activity ->
CommonAPI.repeat(activity.id, user)
end)
Benchee.run(
%{
"user timeline, no followers" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Plug.Conn.assign(:user, reading_user)
|> Plug.Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
},
inputs: %{"user" => user, "no user" => nil},
time: 60
)
users
|> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
Benchee.run(
%{
"user timeline, all following" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Plug.Conn.assign(:user, reading_user)
|> Plug.Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
},
inputs: %{"user" => user, "no user" => nil},
time: 60
)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end
......@@ -2453,7 +2453,7 @@
%{
key: :relations_actions,
type: [:tuple, {:list, :tuple}],
description: "For actions on relations with all users (follow, unfollow)",
description: "For actions on relationships with all users (follow, unfollow)",
suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]]
},
%{
......
......@@ -414,6 +414,54 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
- `nicknames`
- Response: none (code `204`)
## `GET /api/pleroma/admin/users/:nickname/credentials`
### Get the user's email, password, display and settings-related fields
- Params:
- `nickname`
- Response:
```json
{
"actor_type": "Person",
"allow_following_move": true,
"avatar": "https://pleroma.social/media/7e8e7508fd545ef580549b6881d80ec0ff2c81ed9ad37b9bdbbdf0e0d030159d.jpg",
"background": "https://pleroma.social/media/4de34c0bd10970d02cbdef8972bef0ebbf55f43cadc449554d4396156162fe9a.jpg",
"banner": "https://pleroma.social/media/8d92ba2bd244b613520abf557dd448adcd30f5587022813ee9dd068945986946.jpg",
"bio": "bio",
"default_scope": "public",
"discoverable": false,
"email": "user@example.com",
"fields": [
{
"name": "example",
"value": "<a href=\"https://example.com\" rel=\"ugc\">https://example.com</a>"
}
],
"hide_favorites": false,
"hide_followers": false,
"hide_followers_count": false,
"hide_follows": false,
"hide_follows_count": false,
"id": "9oouHaEEUR54hls968",
"locked": true,
"name": "user",
"no_rich_text": true,
"pleroma_settings_store": {},
"raw_fields": [
{
"id": 1,
"name": "example",
"value": "https://example.com"
},
],
"show_role": true,
"skip_thread_containment": false
}
```
## `PATCH /api/pleroma/admin/users/:current_nickname/update_nickname`
### Update nickname for a user with a given nickname
......
......@@ -95,6 +95,17 @@ def with_preloaded_object(query, join_type \\ :inner) do
|> preload([activity, object: object], object: object)
end
# Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
def user_actor(%Activity{actor: nil}), do: nil
def user_actor(%Activity{} = activity) do
with %User{} <- activity.user_actor do
activity.user_actor
else
_ -> User.get_cached_by_ap_id(activity.actor)
end
end
def with_joined_user_actor(query, join_type \\ :inner) do
join(query, join_type, [activity], u in User,
on: u.ap_id == activity.actor,
......
......@@ -35,6 +35,13 @@ def by_author(query \\ Activity, %User{ap_id: ap_id}) do
from(a in query, where: a.actor == ^ap_id)
end
def find_by_object_ap_id(activities, object_ap_id) do
Enum.find(
activities,
&(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]])
)
end
@spec by_object_id(query, String.t() | [String.t()]) :: query
def by_object_id(query \\ Activity, object_id)
......
......@@ -129,21 +129,18 @@ def for_user(user, params \\ %{}) do
end
def restrict_recipients(query, user, %{"recipients" => user_ids}) do
user_ids =
user_binary_ids =
[user.id | user_ids]
|> Enum.uniq()
|> Enum.reduce([], fn user_id, acc ->
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
[user_id | acc]
end)
|> User.binary_id()
conversation_subquery =
__MODULE__
|> group_by([p], p.conversation_id)
|> having(
[p],
count(p.user_id) == ^length(user_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
count(p.user_id) == ^length(user_binary_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
)
|> select([p], %{id: p.conversation_id})
......
......@@ -135,12 +135,8 @@ def all_between_user_sets(
target_users
)
when is_list(source_users) and is_list(target_users) do
get_bin_ids = fn user ->
with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id
end
source_user_ids = Enum.map(source_users, &get_bin_ids.(&1))
target_user_ids = Enum.map(target_users, &get_bin_ids.(&1))
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
__MODULE__
|> where(
......@@ -155,4 +151,10 @@ def all_between_user_sets(
)
|> Repo.all()
end
def find(following_relationships, follower, following) do
Enum.find(following_relationships, fn
fr -> fr.follower_id == follower.id and fr.following_id == following.id
end)
end
end
......@@ -605,6 +605,17 @@ def get_log_entry_message(%ModerationLog{
}"
end
@spec get_log_entry_message(ModerationLog) :: String.t()
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "updated_users",
"subject" => subjects
}
}) do
"@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")
......
......@@ -10,6 +10,7 @@ defmodule Pleroma.Notification do
alias Pleroma.Object
alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.ThreadMute
alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.Push
......@@ -17,6 +18,7 @@ defmodule Pleroma.Notification do
import Ecto.Query
import Ecto.Changeset
require Logger
@type t :: %__MODULE__{}
......@@ -37,11 +39,11 @@ def changeset(%Notification{} = notification, attrs) do
end
defp for_user_query_ap_id_opts(user, opts) do
ap_id_relations =
ap_id_relationships =
[:block] ++
if opts[@include_muted_option], do: [], else: [:notification_mute]
preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations)
preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships)
exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
......@@ -100,7 +102,7 @@ defp exclude_notification_muted(query, user, opts) do
query
|> where([n, a], a.actor not in ^notification_muted_ap_ids)
|> join(:left, [n, a], tm in Pleroma.ThreadMute,
|> join(:left, [n, a], tm in ThreadMute,
on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
)
|> where([n, a, o, tm], is_nil(tm.user_id))
......@@ -275,58 +277,111 @@ def dismiss(%{id: user_id} = _user, id) do
def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
object = Object.normalize(activity)
unless object && object.data["type"] == "Answer" do
users = get_notified_from_activity(activity)
notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
{:ok, notifications}
else
if object && object.data["type"] == "Answer" do
{:ok, []}
else
do_create_notifications(activity)
end
end
def create_notifications(%Activity{data: %{"type" => type}} = activity)
when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
do_create_notifications(activity)
end
def create_notifications(_), do: {:ok, []}
defp do_create_notifications(%Activity{} = activity) do
{enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
potential_receivers = enabled_receivers ++ disabled_receivers
notifications =
activity
|> get_notified_from_activity()
|> Enum.map(&create_notification(activity, &1))
Enum.map(potential_receivers, fn user ->
do_send = user in enabled_receivers
create_notification(activity, user, do_send)
end)
{:ok, notifications}
end
def create_notifications(_), do: {:ok, []}
# TODO move to sql, too.
def create_notification(%Activity{} = activity, %User{} = user) do
def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
unless skip?(activity, user) do
notification = %Notification{user_id: user.id, activity: activity}
{:ok, notification} = Repo.insert(notification)
["user", "user:notification"]
|> Streamer.stream(notification)
if do_send do
Streamer.stream(["user", "user:notification"], notification)
Push.send(notification)
end
Push.send(notification)
notification
end
end
@doc """
Returns a tuple with 2 elements:
{enabled notification receivers, currently disabled receivers (blocking / [thread] muting)}
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
"""
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
|> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
|> User.get_users_from_set(local_only)
potential_receiver_ap_ids =
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
|> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
# Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs
notification_enabled_ap_ids =
potential_receiver_ap_ids
|> exclude_relationship_restricted_ap_ids(activity)
|> exclude_thread_muter_ap_ids(activity)
potential_receivers =
potential_receiver_ap_ids
|> Enum.uniq()
|> User.get_users_from_set(local_only)
notification_enabled_users =
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
{notification_enabled_users, potential_receivers -- notification_enabled_users}
end
def get_notified_from_activity(_, _local_only), do: {[], []}
@doc "Filters out AP IDs of users basing on their relationships with activity actor user"
def exclude_relationship_restricted_ap_ids([], _activity), do: []
def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do
relationship_restricted_ap_ids =
activity
|> Activity.user_actor()
|> User.incoming_relationships_ungrouped_ap_ids([
:block,
:notification_mute
])
Enum.uniq(ap_ids) -- relationship_restricted_ap_ids
end
def get_notified_from_activity(_, _local_only), do: []
@doc "Filters out AP IDs of users who mute activity thread"
def exclude_thread_muter_ap_ids([], _activity), do: []
def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"])
Enum.uniq(ap_ids) -- thread_muter_ap_ids
end
@spec skip?(Activity.t(), User.t()) :: boolean()
def skip?(activity, user) do
def skip?(%Activity{} = activity, %User{} = user) do
[
:self,
:followers,
......@@ -335,18 +390,20 @@ def skip?(activity, user) do
:non_follows,
:recently_followed
]
|> Enum.any?(&skip?(&1, activity, user))
|> Enum.find(&skip?(&1, activity, user))
end
def skip?(_, _), do: false
@spec skip?(atom(), Activity.t(), User.t()) :: boolean()
def skip?(:self, activity, user) do
def skip?(:self, %Activity{} = activity, %User{} = user) do
activity.data["actor"] == user.ap_id
end
def skip?(
:followers,
activity,
%{notification_settings: %{followers: false}} = user
%Activity{} = activity,
%User{notification_settings: %{followers: false}} = user
) do
actor = activity.data["actor"]
follower = User.get_cached_by_ap_id(actor)
......@@ -355,15 +412,19 @@ def skip?(
def skip?(
:non_followers,
activity,
%{notification_settings: %{non_followers: false}} = user
%Activity{} = activity,
%User{notification_settings: %{non_followers: false}} = user
) do
actor = activity.data["actor"]
follower = User.get_cached_by_ap_id(actor)
!User.following?(follower, user)
end
def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do
def skip?(
:follows,
%Activity{} = activity,
%User{notification_settings: %{follows: false}} = user
) do
actor = activity.data["actor"]
followed = User.get_cached_by_ap_id(actor)
User.following?(user, followed)
......@@ -371,15 +432,16 @@ def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user
def skip?(
:non_follows,
activity,
%{notification_settings: %{non_follows: false}} = user
%Activity{} = activity,
%User{notification_settings: %{non_follows: false}} = user
) do
actor = activity.data["actor"]
followed = User.get_cached_by_ap_id(actor)
!User.following?(user, followed)
end
def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do
# To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
actor = activity.data["actor"]
Notification.for_user(user)
......
......@@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do
alias Pleroma.ThreadMute
alias Pleroma.User
require Ecto.Query
import Ecto.Changeset
import Ecto.Query
schema "thread_mutes" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
......@@ -18,19 +19,44 @@ defmodule Pleroma.ThreadMute do
def changeset(mute, params \\ %{}) do
mute
|> Ecto.Changeset.cast(params, [:user_id, :context])
|> Ecto.Changeset.foreign_key_constraint(:user_id)
|> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index)
|> cast(params, [:user_id, :context])
|> foreign_key_constraint(:user_id)
|> unique_constraint(:user_id, name: :unique_index)
end
def query(user_id, context) do
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
user_binary_id = User.binary_id(user_id)
ThreadMute
|> Ecto.Query.where(user_id: ^user_id)
|> Ecto.Query.where(context: ^context)
|> where(user_id: ^user_binary_id)
|> where(context: ^context)
end
def muters_query(context) do
ThreadMute
|> join(:inner, [tm], u in assoc(tm, :user))
|> where([tm], tm.context == ^context)
|> select([tm, u], u.ap_id)
end
def muter_ap_ids(context, ap_ids \\ nil)
# Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
def muter_ap_ids(context, _ap_ids) when is_nil(context), do: []
def muter_ap_ids(context, ap_ids) do
context
|> muters_query()
|> maybe_filter_on_ap_id(ap_ids)
|> Repo.all()
end
defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
where(query, [tm, u], u.ap_id in ^ap_ids)
end
defp maybe_filter_on_ap_id(query, _ap_ids), do: query
def add_mute(user_id, context) do
%ThreadMute{}
|> changeset(%{user_id: user_id, context: context})
......
This diff is collapsed.
......@@ -8,7 +8,7 @@ defmodule Pleroma.UserRelationship do
import Ecto.Changeset
import Ecto.Query
alias FlakeId.Ecto.CompatType
alias Pleroma.FollowingRelationship
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.UserRelationship
......@@ -22,15 +22,18 @@ defmodule Pleroma.UserRelationship do
end
for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do
# Definitions of `create_block/2`, `create_mute/2` etc.
# `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
# `def create_notification_mute/2`, `def create_inverse_subscription/2`
def unquote(:"create_#{relationship_type}")(source, target),
do: create(unquote(relationship_type), source, target)
# Definitions of `delete_block/2`, `delete_mute/2` etc.
# `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
# `def delete_notification_mute/2`, `def delete_inverse_subscription/2`
def unquote(:"delete_#{relationship_type}")(source, target),
do: delete(unquote(relationship_type), source, target)
# Definitions of `block_exists?/2`, `mute_exists?/2` etc.
# `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
# `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`
def unquote(:"#{relationship_type}_exists?")(source, target),
do: exists?(unquote(relationship_type), source, target)
end
......@@ -84,12 +87,8 @@ def dictionary(
target_to_source_rel_types \\ nil
)
when is_list(source_users) and is_list(target_users) do
get_bin_ids = fn user ->
with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id
end
source_user_ids = Enum.map(source_users, &get_bin_ids.(&1))
target_user_ids = Enum.map(target_users, &get_bin_ids.(&1))
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end
......@@ -129,6 +128,25 @@ def exists?(dictionary, rel_type, source, target, func) do
end
end
@doc ":relationships option for StatusView / AccountView / NotificationView"
def view_relationships_option(nil = _reading_user, _actors) do
%{user_relationships: [], following_relationships: []}
end