Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into 1507-static-fe-prioritize-json

.gitlab/issue_templates/Bug.md

@@ -14,7 +14,7 @@
 * Pleroma version (could be found in the "Version" tab of settings in Pleroma-FE): 
 * Elixir version (`elixir -v` for from source installations, N/A for OTP):
 * Operating system:
-* PostgreSQL version (`postgres -V`):
+* PostgreSQL version (`psql -V`):
 
 
 ### Bug description

CHANGELOG.md

@@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   <summary>API Changes</summary>
 
 - **Breaking:** Emoji API: changed methods and renamed routes.
+- Streaming: Repeats of a user's posts will no longer be pushed to the user's stream.
+- Mastodon API: Added `pleroma.metadata.fields_limits` to /api/v1/instance
+- Mastodon API: On deletion, returns the original post text.
 </details>
 
 <details>
@@ -51,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 <details>
   <summary>API Changes</summary>
+- Mastodon API: Add pleroma.parents_visible field to statuses.
 - Mastodon API: Extended `/api/v1/instance`.
 - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
 - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.

config/config.exs

@@ -437,8 +437,6 @@
 config :pleroma, Pleroma.Web.Preload,
   providers: [
     Pleroma.Web.Preload.Providers.Instance,
-    Pleroma.Web.Preload.Providers.User,
-    Pleroma.Web.Preload.Providers.Timelines,
     Pleroma.Web.Preload.Providers.StatusNet
   ]
 

docs/API/differences_in_mastoapi_responses.md

@@ -27,6 +27,7 @@ Has these additional fields under the `pleroma` object:
 - `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
+- `parent_visible`: If the parent of this post is visible to the user or not.
 
 ## Media Attachments
 
@@ -51,11 +52,14 @@ The `id` parameter can also be the `nickname` of the user. This only works in th
 
 Has these additional fields under the `pleroma` object:
 
+- `ap_id`: nullable URL string, ActivityPub id of the user
+- `background_image`: nullable URL string, background image of the user
 - `tags`: Lists an array of tags for the user
-- `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/entities/relationship/
+- `relationship` (object): Includes fields as documented for Mastodon API https://docs.joinmastodon.org/entities/relationship/
 - `is_moderator`: boolean, nullable,  true if user is a moderator
 - `is_admin`: boolean, nullable, true if user is an admin
 - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated
+- `hide_favorites`: boolean, true when the user has hiding favorites enabled
 - `hide_followers`: boolean, true when the user has follower hiding enabled
 - `hide_follows`: boolean, true when the user has follow hiding enabled
 - `hide_followers_count`: boolean, true when the user has follower stat hiding enabled
@@ -66,6 +70,7 @@ Has these additional fields under the `pleroma` object:
 - `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts
 - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
 - `unread_notifications_count`: The count of unread notifications. Only returned to the account owner.
+- `notification_settings`: object, can be absent. See `/api/pleroma/notification_settings` for the parameters/keys returned.
 
 ### Source
 
@@ -223,6 +228,7 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `background_image`: A background image that frontends can use
 - `pleroma.metadata.features`: A list of supported features
 - `pleroma.metadata.federation`: The federation restrictions of this instance
+- `pleroma.metadata.fields_limits`: A list of values detailing the length and count limitation for various instance-configurable fields.
 - `vapid_public_key`: The public key needed for push messages
 
 ## Markers

lib/pleroma/config/loader.ex

@@ -12,6 +12,11 @@ defmodule Pleroma.Config.Loader do
     :swarm
   ]
 
+  @reject_groups [
+    :postgrex,
+    :tesla
+  ]
+
   if Code.ensure_loaded?(Config.Reader) do
     @reader Config.Reader
 
@@ -47,7 +52,8 @@ defp filter(configs) do
   @spec filter_group(atom(), keyword()) :: keyword()
   def filter_group(group, configs) do
     Enum.reject(configs[group], fn {key, _v} ->
-      key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex
+      key in @reject_keys or group in @reject_groups or
+        (group == :phoenix and key == :serve_endpoints)
     end)
   end
 end

lib/pleroma/html.ex

@@ -109,7 +109,7 @@ def extract_first_external_url(object, content) do
       result =
         content
         |> Floki.parse_fragment!()
-        |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]")
+        |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]")
         |> Floki.attribute("a", "href")
         |> Enum.at(0)
 

lib/pleroma/migration_helper/notification_backfill.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MigrationHelper.NotificationBackfill do
-  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -25,18 +24,27 @@ def fill_in_notification_types do
         |> type_from_activity()
 
       notification
-      |> Notification.changeset(%{type: type})
+      |> Ecto.Changeset.change(%{type: type})
       |> Repo.update()
     end)
   end
 
+  defp get_by_ap_id(ap_id) do
+    q =
+      from(u in User,
+        select: u.id
+      )
+
+    Repo.get_by(q, ap_id: ap_id)
+  end
+
   # This is copied over from Notifications to keep this stable.
   defp type_from_activity(%{data: %{"type" => type}} = activity) do
     case type do
       "Follow" ->
         accepted_function = fn activity ->
-          with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]),
-               %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do
+          with %User{} = follower <- get_by_ap_id(activity.data["actor"]),
+               %User{} = followed <- get_by_ap_id(activity.data["object"]) do
             Pleroma.FollowingRelationship.following?(follower, followed)
           end
         end

lib/pleroma/notification.ex

@@ -367,6 +367,7 @@ defp do_create_notifications(%Activity{} = activity, options) do
         do_send = do_send && user in enabled_receivers
         create_notification(activity, user, do_send)
       end)
+      |> Enum.reject(&is_nil/1)
 
     {:ok, notifications}
   end

lib/pleroma/object/fetcher.ex

@@ -83,8 +83,8 @@ def fetch_object_from_id(id, options \\ []) do
       {:transmogrifier, {:error, {:reject, nil}}} ->
         {:reject, nil}
 
-      {:transmogrifier, _} ->
-        {:error, "Transmogrifier failure."}
+      {:transmogrifier, _} = e ->
+        {:error, e}
 
       {:object, data, nil} ->
         reinject_object(%Object{}, data)

lib/pleroma/user.ex

@@ -115,7 +115,7 @@ defmodule Pleroma.User do
     field(:is_moderator, :boolean, default: false)
     field(:is_admin, :boolean, default: false)
     field(:show_role, :boolean, default: true)
-    field(:settings, :map, default: nil)
+    field(:mastofe_settings, :map, default: nil)
     field(:uri, ObjectValidators.Uri, default: nil)
     field(:hide_followers_count, :boolean, default: false)
     field(:hide_follows_count, :boolean, default: false)
@@ -1309,7 +1309,8 @@ def block(%User{} = blocker, %User{} = blocked) do
 
     unsubscribe(blocked, blocker)
 
-    if following?(blocked, blocker), do: unfollow(blocked, blocker)
+    unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true)
+    if unfollowing_blocked && following?(blocked, blocker), do: unfollow(blocked, blocker)
 
     {:ok, blocker} = update_follower_count(blocker)
     {:ok, blocker, _} = Participation.mark_all_as_read(blocker, blocked)
@@ -1527,8 +1528,7 @@ def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
       blocked_identifiers,
       fn blocked_identifier ->
         with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
-             {:ok, _user_block} <- block(blocker, blocked),
-             {:ok, _} <- ActivityPub.block(blocker, blocked) do
+             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
           blocked
         else
           err ->
@@ -2118,8 +2118,8 @@ def mascot_update(user, url) do
 
   def mastodon_settings_update(user, settings) do
     user
-    |> cast(%{settings: settings}, [:settings])
-    |> validate_required([:settings])
+    |> cast(%{mastofe_settings: settings}, [:mastofe_settings])
+    |> validate_required([:mastofe_settings])
     |> update_and_set_cache()
   end
 

lib/pleroma/user/search.ex

@@ -52,6 +52,7 @@ defp search_query(query_string, for_user, following) do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
+    |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
     |> trigram_rank(query_string)
@@ -109,6 +110,10 @@ defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
   end
 
+  defp filter_internal_users(query) do
+    from(q in query, where: q.actor_type != "Application")
+  end
+
   defp filter_blocked_user(query, %User{} = blocker) do
     query
     |> join(:left, [u], b in Pleroma.UserRelationship,

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -366,33 +366,6 @@ defp do_unfollow(follower, followed, activity_id, local) do
     end
   end
 
-  @spec block(User.t(), User.t(), String.t() | nil, boolean()) ::
-          {:ok, Activity.t()} | {:error, any()}
-  def block(blocker, blocked, activity_id \\ nil, local \\ true) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_block(blocker, blocked, activity_id, local) end) do
-      result
-    end
-  end
-
-  defp do_block(blocker, blocked, activity_id, local) do
-    unfollow_blocked = Config.get([:activitypub, :unfollow_blocked])
-
-    if unfollow_blocked and fetch_latest_follow(blocker, blocked) do
-      unfollow(blocker, blocked, nil, local)
-    end
-
-    block_data = make_block_data(blocker, blocked, activity_id)
-
-    with {:ok, activity} <- insert(block_data, local),
-         _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    else
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
   def flag(
         %{
@@ -1398,6 +1371,16 @@ def fetch_and_prepare_user_from_ap_id(ap_id) do
     end
   end
 
+  def maybe_handle_clashing_nickname(nickname) do
+    with %User{} = old_user <- User.get_by_nickname(nickname) do
+      Logger.info("Found an old user for #{nickname}, ap id is #{old_user.ap_id}, renaming.")
+
+      old_user
+      |> User.remote_user_changeset(%{nickname: "#{old_user.id}.#{old_user.nickname}"})
+      |> User.update_and_set_cache()
+    end
+  end
+
   def make_user_from_ap_id(ap_id) do
     user = User.get_cached_by_ap_id(ap_id)
 
@@ -1410,6 +1393,8 @@ def make_user_from_ap_id(ap_id) do
           |> User.remote_user_changeset(data)
           |> User.update_and_set_cache()
         else
+          maybe_handle_clashing_nickname(data[:nickname])
+
           data
           |> User.remote_user_changeset()
           |> Repo.insert()

lib/pleroma/web/activity_pub/builder.ex

@@ -138,6 +138,18 @@ def update(actor, object) do
      }, []}
   end
 
+  @spec block(User.t(), User.t()) :: {:ok, map(), keyword()}
+  def block(blocker, blocked) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "type" => "Block",
+       "actor" => blocker.ap_id,
+       "object" => blocked.ap_id,
+       "to" => [blocked.ap_id]
+     }, []}
+  end
+
   @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()}
   def announce(actor, object, options \\ []) do
     public? = Keyword.get(options, :public, false)

lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex

@@ -27,11 +27,14 @@ defp contains_links?(_), do: false
 
   @impl true
   def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do
-    with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor),
+    with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
          {:contains_links, true} <- {:contains_links, contains_links?(object)},
          {:old_user, true} <- {:old_user, old_user?(u)} do
       {:ok, message}
     else
+      {:ok, %User{local: true}} ->
+        {:ok, message}
+
       {:contains_links, false} ->
         {:ok, message}
 

lib/pleroma/web/activity_pub/object_validator.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
@@ -24,6 +25,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)
 
+  def validate(%{"type" => "Block"} = block_activity, meta) do
+    with {:ok, block_activity} <-
+           block_activity
+           |> BlockValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      block_activity = stringify_keys(block_activity)
+      outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks])
+
+      meta =
+        if !outgoing_blocks do
+          Keyword.put(meta, :do_not_federate, true)
+        else
+          meta
+        end
+
+      {:ok, block_activity, meta}
+    end
+  end
+
   def validate(%{"type" => "Update"} = update_activity, meta) do
     with {:ok, update_activity} <-
            update_activity

lib/pleroma/web/activity_pub/object_validators/block_validator.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:type, :string)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:object, ObjectValidators.ObjectID)
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(cng) do
+    cng
+    |> validate_required([:id, :type, :actor, :to, :cc, :object])
+    |> validate_inclusion(:type, ["Block"])
+    |> validate_actor_presence()
+    |> validate_actor_presence(field_name: :object)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data
+    |> validate_data
+  end
+end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   collection, and so on.
   """
   alias Pleroma.Activity
+  alias Pleroma.Activity.Ir.Topics
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.Notification
@@ -20,6 +21,21 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
   def handle(object, meta \\ [])
 
+  # Tasks this handles:
+  # - Unfollow and block
+  def handle(
+        %{data: %{"type" => "Block", "object" => blocked_user, "actor" => blocking_user}} =
+          object,
+        meta
+      ) do
+    with %User{} = blocker <- User.get_cached_by_ap_id(blocking_user),
+         %User{} = blocked <- User.get_cached_by_ap_id(blocked_user) do
+      User.block(blocker, blocked)
+    end
+
+    {:ok, object, meta}
+  end
+
   # Tasks this handles:
   # - Update the user
   #
@@ -82,7 +98,10 @@ def handle(%{data: %{"type" => "Announce"}} = object, meta) do
 
     if !User.is_internal_user?(user) do
       Notification.create_notifications(object)
-      ActivityPub.stream_out(object)
+
+      object
+      |> Topics.get_activity_topics()
+      |> Streamer.stream(object)
     end
 
     {:ok, object, meta}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -446,12 +446,9 @@ def handle_incoming(
       when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
     actor = Containment.get_actor(data)
 
-    data =
-      Map.put(data, "actor", actor)
-      |> fix_addressing
-
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor),
+         data <- Map.put(data, "actor", actor) |> fix_addressing() do
       object = fix_object(object, options)
 
       params = %{
@@ -673,7 +670,7 @@ def handle_incoming(
   end
 
   def handle_incoming(%{"type" => type} = data, _options)
-      when type in ["Like", "EmojiReact", "Announce"] do
+      when type in ~w{Like EmojiReact Announce} do
     with :ok <- ObjectValidator.fetch_actor_and_object(data),
          {:ok, activity, _meta} <-
            Pipeline.common_pipeline(data, local: false) do
@@ -684,9 +681,10 @@ def handle_incoming(%{"type" => type} = data, _options)
   end
 
   def handle_incoming(
-        %{"type" => "Update"} = data,
+        %{"type" => type} = data,
         _options
-      ) do
+      )
+      when type in ~w{Update Block} do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
@@ -765,21 +763,6 @@ def handle_incoming(
     end
   end
 
-  def handle_incoming(
-        %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data,
-        _options
-      ) do
-    with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker),
-         {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
-      User.unfollow(blocker, blocked)
-      User.block(blocker, blocked)
-      {:ok, activity}
-    else
-      _e -> :error
-    end
-  end
-
   def handle_incoming(
         %{
           "type" => "Move",

lib/pleroma/web/activity_pub/visibility.ex

@@ -47,6 +47,10 @@ def is_list?(_), do: false
   @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean()
   def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
 
+  def visible_for_user?(nil, _), do: false
+
+  def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
+
   def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
     user.ap_id in activity.data["to"] ||
       list_ap_id
@@ -54,8 +58,6 @@ def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{
       |> Pleroma.List.member?(user)
   end
 
-  def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
-
   def visible_for_user?(%{local: local} = activity, nil) do
     cfg_key =
       if local,