Commit 349b9d86 authored by lain's avatar lain

Merge branch 'remake-remodel-2' into 'develop'

Ingestion Pipeline Revamp

See merge request !2315
parents 8444e7ee 772bc258
Pipeline #24216 passed with stages
in 48 minutes and 55 seconds
Unless otherwise stated this repository is copyright © 2017-2019
Unless otherwise stated this repository is copyright © 2017-2020
Pleroma Authors <https://pleroma.social/>, and is distributed under
The GNU Affero General Public License Version 3, you should have received a
copy of the license file as AGPL-3.
......@@ -23,7 +23,7 @@ priv/static/images/pleroma-fox-tan-shy.png
---
The following files are copyright © 2017-2019 Pleroma Authors
The following files are copyright © 2017-2020 Pleroma Authors
<https://pleroma.social/>, and are distributed under the Creative Commons
Attribution-ShareAlike 4.0 International license, you should have received
a copy of the license file as CC-BY-SA-4.0.
......
......@@ -32,6 +32,18 @@ def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor)
get_actor(%{"actor" => actor})
end
def get_object(%{"object" => id}) when is_binary(id) do
id
end
def get_object(%{"object" => %{"id" => id}}) when is_binary(id) do
id
end
def get_object(_) do
nil
end
# TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
# objects being present in the test suite environment. Once these objects are
# removed, please also remove this.
......
......@@ -125,6 +125,21 @@ def increase_poll_votes_if_vote(%{
def increase_poll_votes_if_vote(_create_data), do: :noop
@spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
def persist(object, meta) do
with local <- Keyword.fetch!(meta, :local),
{recipients, _, _} <- get_recipients(object),
{:ok, activity} <-
Repo.insert(%Activity{
data: object,
local: local,
recipients: recipients,
actor: object["actor"]
}) do
{:ok, activity, meta}
end
end
@spec insert(map(), boolean(), boolean(), boolean()) :: {:ok, Activity.t()} | {:error, any()}
def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do
with nil <- Activity.normalize(map),
......
defmodule Pleroma.Web.ActivityPub.Builder do
@moduledoc """
This module builds the objects. Meant to be used for creating local objects.
This module encodes our addressing policies and general shape of our objects.
"""
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
@spec like(User.t(), Object.t()) :: {:ok, map(), keyword()}
def like(actor, object) do
object_actor = User.get_cached_by_ap_id(object.data["actor"])
# Address the actor of the object, and our actor's follower collection if the post is public.
to =
if Visibility.is_public?(object) do
[actor.follower_address, object.data["actor"]]
else
[object.data["actor"]]
end
# CC everyone who's been addressed in the object, except ourself and the object actor's
# follower collection
cc =
(object.data["to"] ++ (object.data["cc"] || []))
|> List.delete(actor.ap_id)
|> List.delete(object_actor.follower_address)
{:ok,
%{
"id" => Utils.generate_activity_id(),
"actor" => actor.ap_id,
"type" => "Like",
"object" => object.data["id"],
"to" => to,
"cc" => cc,
"context" => object.data["context"]
}, []}
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidator do
@moduledoc """
This module is responsible for validating an object (which can be an activity)
and checking if it is both well formed and also compatible with our view of
the system.
"""
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
@spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
def validate(object, meta)
def validate(%{"type" => "Like"} = object, meta) do
with {:ok, object} <-
object |> LikeValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object |> Map.from_struct())
{:ok, object, meta}
end
end
def stringify_keys(object) do
object
|> Map.new(fn {key, val} -> {to_string(key), val} end)
end
def fetch_actor_and_object(object) do
User.get_or_fetch_by_ap_id(object["actor"])
Object.normalize(object["object"])
:ok
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
import Ecto.Changeset
alias Pleroma.Object
alias Pleroma.User
def validate_actor_presence(cng, field_name \\ :actor) do
cng
|> validate_change(field_name, fn field_name, actor ->
if User.get_cached_by_ap_id(actor) do
[]
else
[{field_name, "can't find user"}]
end
end)
end
def validate_object_presence(cng, field_name \\ :object) do
cng
|> validate_change(field_name, fn field_name, object ->
if Object.get_cached_by_ap_id(object) do
[]
else
[{field_name, "can't find object"}]
end
end)
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
use Ecto.Schema
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
@primary_key false
embedded_schema do
field(:id, Types.ObjectID, primary_key: true)
field(:actor, Types.ObjectID)
field(:type, :string)
field(:to, {:array, :string})
field(:cc, {:array, :string})
field(:bto, {:array, :string}, default: [])
field(:bcc, {:array, :string}, default: [])
embeds_one(:object, NoteValidator)
end
def cast_data(data) do
cast(%__MODULE__{}, data, __schema__(:fields))
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
use Ecto.Schema
alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.Utils
import Ecto.Changeset
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@primary_key false
embedded_schema do
field(:id, Types.ObjectID, primary_key: true)
field(:type, :string)
field(:object, Types.ObjectID)
field(:actor, Types.ObjectID)
field(:context, :string)
field(:to, {:array, :string})
field(:cc, {:array, :string})
end
def cast_and_validate(data) do
data
|> cast_data()
|> validate_data()
end
def cast_data(data) do
%__MODULE__{}
|> cast(data, [:id, :type, :object, :actor, :context, :to, :cc])
end
def validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Like"])
|> validate_required([:id, :type, :object, :actor, :context, :to, :cc])
|> validate_actor_presence()
|> validate_object_presence()
|> validate_existing_like()
end
def validate_existing_like(%{changes: %{actor: actor, object: object}} = cng) do
if Utils.get_existing_like(actor, %{data: %{"id" => object}}) do
cng
|> add_error(:actor, "already liked this object")
|> add_error(:object, "already liked by this actor")
else
cng
end
end
def validate_existing_like(cng), do: cng
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
use Ecto.Schema
alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
@primary_key false
embedded_schema do
field(:id, Types.ObjectID, primary_key: true)
field(:to, {:array, :string}, default: [])
field(:cc, {:array, :string}, default: [])
field(:bto, {:array, :string}, default: [])
field(:bcc, {:array, :string}, default: [])
# TODO: Write type
field(:tag, {:array, :map}, default: [])
field(:type, :string)
field(:content, :string)
field(:context, :string)
field(:actor, Types.ObjectID)
field(:attributedTo, Types.ObjectID)
field(:summary, :string)
field(:published, Types.DateTime)
# TODO: Write type
field(:emoji, :map, default: %{})
field(:sensitive, :boolean, default: false)
# TODO: Write type
field(:attachment, {:array, :map}, default: [])
field(:replies_count, :integer, default: 0)
field(:like_count, :integer, default: 0)
field(:announcement_count, :integer, default: 0)
field(:inRepyTo, :string)
field(:likes, {:array, :string}, default: [])
field(:announcements, {:array, :string}, default: [])
# see if needed
field(:conversation, :string)
field(:context_id, :string)
end
def cast_and_validate(data) do
data
|> cast_data()
|> validate_data()
end
def cast_data(data) do
%__MODULE__{}
|> cast(data, __schema__(:fields))
end
def validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Note"])
|> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
end
end
defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do
@moduledoc """
The AP standard defines the date fields in AP as xsd:DateTime. Elixir's
DateTime can't parse this, but it can parse the related iso8601. This
module punches the date until it looks like iso8601 and normalizes to
it.
DateTimes without a timezone offset are treated as UTC.
Reference: https://www.w3.org/TR/activitystreams-vocabulary/#dfn-published
"""
use Ecto.Type
def type, do: :string
def cast(datetime) when is_binary(datetime) do
with {:ok, datetime, _} <- DateTime.from_iso8601(datetime) do
{:ok, DateTime.to_iso8601(datetime)}
else
{:error, :missing_offset} -> cast("#{datetime}Z")
_e -> :error
end
end
def cast(_), do: :error
def dump(data) do
{:ok, data}
end
def load(data) do
{:ok, data}
end
end
defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do
use Ecto.Type
def type, do: :string
def cast(object) when is_binary(object) do
# Host has to be present and scheme has to be an http scheme (for now)
case URI.parse(object) do
%URI{host: nil} ->
:error
%URI{scheme: scheme} when scheme in ["https", "http"] ->
{:ok, object}
_ ->
:error
end
end
def cast(%{"id" => object}), do: cast(object)
def cast(_) do
:error
end
def dump(data) do
{:ok, data}
end
def load(data) do
{:ok, data}
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.Pipeline do
alias Pleroma.Activity
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.MRF
alias Pleroma.Web.ActivityPub.ObjectValidator
alias Pleroma.Web.ActivityPub.SideEffects
alias Pleroma.Web.Federator
@spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()}
def common_pipeline(object, meta) do
with {_, {:ok, validated_object, meta}} <-
{:validate_object, ObjectValidator.validate(object, meta)},
{_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
{_, {:ok, %Activity{} = activity, meta}} <-
{:persist_object, ActivityPub.persist(mrfd_object, meta)},
{_, {:ok, %Activity{} = activity, meta}} <-
{:execute_side_effects, SideEffects.handle(activity, meta)},
{_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
{:ok, activity, meta}
else
{:mrf_object, {:reject, _}} -> {:ok, nil, meta}
e -> {:error, e}
end
end
defp maybe_federate(activity, meta) do
with {:ok, local} <- Keyword.fetch(meta, :local) do
if local do
Federator.publish(activity)
{:ok, :federated}
else
{:ok, :not_federated}
end
else
_e -> {:error, :badarg}
end
end
end
defmodule Pleroma.Web.ActivityPub.SideEffects do
@moduledoc """
This module looks at an inserted object and executes the side effects that it
implies. For example, a `Like` activity will increase the like count on the
liked object, a `Follow` activity will add the user to the follower
collection, and so on.
"""
alias Pleroma.Notification
alias Pleroma.Object
alias Pleroma.Web.ActivityPub.Utils
def handle(object, meta \\ [])
# Tasks this handles:
# - Add like to object
# - Set up notification
def handle(%{data: %{"type" => "Like"}} = object, meta) do
liked_object = Object.get_by_ap_id(object.data["object"])
Utils.add_like_to_object(object, liked_object)
Notification.create_notifications(object)
{:ok, object, meta}
end
# Nothing to do
def handle(object, meta) do
{:ok, object, meta}
end
end
......@@ -13,6 +13,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.ObjectValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Web.Federator
......@@ -609,17 +612,20 @@ def handle_incoming(
|> handle_incoming(options)
end
def handle_incoming(
%{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data,
_options
) do
with actor <- Containment.get_actor(data),
{:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
{:ok, object} <- get_obj_helper(object_id),
{:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
def handle_incoming(%{"type" => "Like"} = data, _options) do
with {_, {:ok, cast_data_sym}} <-
{:casting_data,
data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)},
cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)),
:ok <- ObjectValidator.fetch_actor_and_object(cast_data),
{_, {:ok, cast_data}} <- {:ensure_context_presence, ensure_context_presence(cast_data)},
{_, {:ok, cast_data}} <-
{:ensure_recipients_presence, ensure_recipients_presence(cast_data)},
{_, {:ok, activity, _meta}} <-
{:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do
{:ok, activity}
else
_e -> :error
e -> {:error, e}
end
end
......@@ -1243,4 +1249,45 @@ def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do
def maybe_fix_user_url(data), do: data
def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
defp ensure_context_presence(%{"context" => context} = data) when is_binary(context),
do: {:ok, data}
defp ensure_context_presence(%{"object" => object} = data) when is_binary(object) do
with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do
{:ok, Map.put(data, "context", context)}
else
_ ->
{:error, :no_context}
end
end
defp ensure_context_presence(_) do
{:error, :no_context}
end
defp ensure_recipients_presence(%{"to" => [_ | _], "cc" => [_ | _]} = data),
do: {:ok, data}
defp ensure_recipients_presence(%{"object" => object} = data) do
case Object.normalize(object) do
%{data: %{"actor" => actor}} ->
data =
data
|> Map.put("to", [actor])
|> Map.put("cc", data["cc"] || [])
{:ok, data}
nil ->
{:error, :no_object}
_ ->
{:error, :no_actor}
end
end
defp ensure_recipients_presence(_) do
{:error, :no_object}
end
end
......@@ -12,6 +12,8 @@ defmodule Pleroma.Web.CommonAPI do
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
......@@ -19,6 +21,7 @@ defmodule Pleroma.Web.CommonAPI do
import Pleroma.Web.CommonAPI.Utils
require Pleroma.Constants
require Logger
def follow(follower, followed) do
timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout])
......@@ -109,18 +112,51 @@ def unrepeat(id_or_ap_id, user) do
end
end
def favorite(id_or_ap_id, user) do
with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)},
object <- Object.normalize(activity),
like_activity <- Utils.get_existing_like(user.ap_id, object) do
if like_activity do
{:ok, like_activity, object}
else
ActivityPub.like(user, object)
end
@spec favorite(User.t(), binary()) :: {:ok, Activity.t() | :already_liked} | {:error, any()}
def favorite(%User{} = user, id) do
case favorite_helper(user, id) do
{:ok, _} = res ->
res
{:error, :not_found} = res ->
res
{:error, e} ->
Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}")
{:error, dgettext("errors", "Could not favorite")}
end
end
def favorite_helper(user, id) do
with {_, %Activity{object: object}} <- {:find_object, Activity.get_by_id_with_object(id)},
{_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)},
{_, {:ok, %Activity{} = activity, _meta}} <-
{:common_pipeline,
Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do
{:ok, activity}
else
{:find_activity, _} -> {:error, :not_found}
_ -> {:error, dgettext("errors", "Could not favorite")}
{:find_object, _} ->
{:error, :not_found}
{:common_pipeline,
{
:error,
{
:validate_object,
{
:error,
changeset
}
}
}} = e ->
if {:object, {"already liked by this actor", []}} in changeset.errors do
{:ok, :already_liked}
else
{:error, e}
end
e ->
{:error, e}
end
end
......
......@@ -207,9 +207,9 @@ def unreblog(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
end
@doc "POST /api/v1/statuses/:id/favourite"
def favourite(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
def favourite(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) do
with {:ok, _fav} <- CommonAPI.favorite(user, activity_id),
%Activity{} = activity <- Activity.get_by_id(activity_id) do
try_render(conn, "show.json", activity: activity, for: user, as: :activity)
end
end
......
......@@ -537,7 +537,7 @@ test "it does not send notification to mentioned users in likes" do
"status" => "hey @#{other_user.nickname}!"
})
{:ok, activity_two, _} = CommonAPI.favorite(activity_one.id, third_user)
{:ok, activity_two} = CommonAPI.favorite(third_user, activity_one.id)
{enabled_receivers, _disabled_receivers} =
Notification.get_notified_from_activity(activity_two)
......@@ -620,7 +620,7 @@ test "liking an activity results in 1 notification, then 0 if the activity is de
assert Enum.empty?(Notification.for_user(user))
{:ok, _, _} = CommonAPI.favorite(activity.id, other_user)
{:ok, _} = CommonAPI.favorite(other_user, activity.id)
assert length(Notification.for_user(user)) == 1