Verified Commit c9449326 authored by Haelwenn's avatar Haelwenn
Browse files

Pipeline Ingestion: Note

parent e2a3365b
......@@ -13,20 +13,23 @@ def cast(object) when is_binary(object) do
cast([object])
end
def cast(object) when is_map(object) do
case ObjectID.cast(object) do
{:ok, data} -> {:ok, data}
_ -> :error
end
end
def cast(data) when is_list(data) do
data
|> Enum.reduce_while({:ok, []}, fn
nil, {:ok, list} ->
{:cont, {:ok, list}}
element, {:ok, list} ->
case ObjectID.cast(element) do
{:ok, id} ->
{:cont, {:ok, [id | list]}}
_ ->
{:halt, {:error, element}}
end
|> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
case ObjectID.cast(element) do
{:ok, id} ->
{:cont, {:ok, [id | list]}}
_ ->
{:cont, {:ok, list}}
end
end)
end
......
......@@ -88,7 +88,7 @@ defp increase_replies_count_if_reply(%{
defp increase_replies_count_if_reply(_create_data), do: :noop
@object_types ~w[ChatMessage Question Answer Audio Video Event Article]
@object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
@impl true
def persist(%{"type" => type} = object, meta) when type in @object_types do
with {:ok, object} <- Object.create(object) do
......
......@@ -101,7 +101,7 @@ def validate(
%{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
meta
)
when objtype in ~w[Question Answer Audio Video Event Article] do
when objtype in ~w[Question Answer Audio Video Event Article Note] do
with {:ok, object_data} <- cast_and_apply(object),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
{:ok, create_activity} <-
......@@ -114,7 +114,7 @@ def validate(
end
def validate(%{"type" => type} = object, meta)
when type in ~w[Event Question Audio Video Article] do
when type in ~w[Event Question Audio Video Article Note] do
validator =
case type do
"Event" -> EventValidator
......@@ -122,6 +122,7 @@ def validate(%{"type" => type} = object, meta)
"Audio" -> AudioVideoValidator
"Video" -> AudioVideoValidator
"Article" -> ArticleNoteValidator
"Note" -> ArticleNoteValidator
end
with {:ok, object} <-
......@@ -183,7 +184,7 @@ def cast_and_apply(%{"type" => "Event"} = object) do
EventValidator.cast_and_apply(object)
end
def cast_and_apply(%{"type" => "Article"} = object) do
def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do
ArticleNoteValidator.cast_and_apply(object)
end
......
......@@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
end
def cast_and_apply(data) do
......@@ -65,24 +67,39 @@ def cast_and_validate(data) do
end
def cast_data(data) do
data = fix(data)
%__MODULE__{}
|> changeset(data)
end
defp fix_url(%{"url" => url} = data) when is_map(url) do
Map.put(data, "url", url["href"])
end
defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data
defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
defp fix_url(data), do: data
defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
defp fix_tag(data), do: Map.drop(data, ["tag"])
defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
do: Map.drop(data, ["replies"])
defp fix_replies(data), do: data
defp fix(data) do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> fix_url()
|> fix_tag()
|> fix_replies()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
end
def changeset(struct, data) do
......
......@@ -26,14 +26,20 @@ def fix_object_defaults(data) do
|> Transmogrifier.fix_implicit_addressing(follower_collection)
end
def fix_activity_defaults(data, meta) do
defp fix_activity_recipients(activity, field, object) do
{:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field])
Map.put(activity, field, data)
end
def fix_activity_defaults(activity, meta) do
object = meta[:object_data] || %{}
data
|> Map.put_new("to", object["to"] || [])
|> Map.put_new("cc", object["cc"] || [])
|> Map.put_new("bto", object["bto"] || [])
|> Map.put_new("bcc", object["bcc"] || [])
activity
|> fix_activity_recipients("to", object)
|> fix_activity_recipients("cc", object)
|> fix_activity_recipients("bto", object)
|> fix_activity_recipients("bcc", object)
end
def fix_actor(data) do
......
......@@ -14,6 +14,7 @@ def validate_any_presence(cng, fields) do
fields
|> Enum.map(fn field -> get_field(cng, field) end)
|> Enum.any?(fn
nil -> false
[] -> false
_ -> true
end)
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
use Ecto.Schema
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
import Ecto.Changeset
@primary_key false
embedded_schema do
field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:actor, ObjectValidators.ObjectID)
field(:type, :string)
field(:to, ObjectValidators.Recipients, default: [])
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
embeds_one(:object, NoteValidator)
end
def cast_data(data) do
cast(%__MODULE__{}, data, __schema__(:fields))
end
end
......@@ -203,6 +203,19 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
Object.increase_replies_count(in_reply_to)
end
reply_depth = (meta[:depth] || 0) + 1
# FIXME: Force inReplyTo to replies
if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and
object.data["replies"] != nil do
for reply_id <- object.data["replies"] do
Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
"id" => reply_id,
"depth" => reply_depth
})
end
end
ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
......@@ -366,7 +379,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
end
def handle_object_creation(%{"type" => objtype} = object, meta)
when objtype in ~w[Audio Video Question Event Article] do
when objtype in ~w[Audio Video Question Event Article Note] do
with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
{:ok, object, meta}
end
......
......@@ -404,10 +404,9 @@ def handle_incoming(%{"id" => id}, _options) when is_binary(id) and byte_size(id
# - tags
# - emoji
def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
%{"type" => "Create", "object" => %{"type" => "Page"} = object} = data,
options
)
when objtype in ~w{Note Page} do
) do
actor = Containment.get_actor(data)
with nil <- Activity.get_create_by_object_ap_id(object["id"]),
......@@ -499,14 +498,15 @@ def handle_incoming(
def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
_options
options
)
when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do
data = Map.put(data, "object", strip_internal_fields(data["object"]))
options = Keyword.put(options, :local, false)
with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
nil <- Activity.get_create_by_object_ap_id(obj_id),
{:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
{:ok, activity, _} <- Pipeline.common_pipeline(data, options) do
{:ok, activity}
else
%Activity{} = activity -> {:ok, activity}
......
......@@ -96,6 +96,11 @@ def perform(:incoming_ap_doc, params) do
Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
{:error, e}
{:error, {:validate_object, _}} = e ->
Logger.error("Incoming AP doc validation error: #{inspect(e)}")
Logger.debug(Jason.encode!(params, pretty: true))
e
e ->
# Just drop those for now
Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
......
......@@ -3,6 +3,7 @@
"type": "Create",
"object": {
"type": "Note",
"to": ["https://www.w3.org/ns/activitystreams#Public"],
"content": "It's a note"
},
"to": ["https://www.w3.org/ns/activitystreams#Public"]
......
......@@ -123,7 +123,8 @@ test "when association is not loaded" do
"type" => "Note",
"content" => "find me!",
"id" => "http://mastodon.example.org/users/admin/objects/1",
"attributedTo" => "http://mastodon.example.org/users/admin"
"attributedTo" => "http://mastodon.example.org/users/admin",
"to" => ["https://www.w3.org/ns/activitystreams#Public"]
},
"to" => ["https://www.w3.org/ns/activitystreams#Public"]
}
......@@ -132,6 +133,7 @@ test "when association is not loaded" do
{:ok, japanese_activity} = Pleroma.Web.CommonAPI.post(user, %{status: "更新情報"})
{:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params)
{:ok, remote_activity} = ObanHelpers.perform(job)
remote_activity = Activity.get_by_id_with_object(remote_activity.id)
%{
japanese_activity: japanese_activity,
......
......@@ -6,10 +6,10 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.RecipientsTest do
alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients
use Pleroma.DataCase, async: true
test "it asserts that all elements of the list are object ids" do
test "it only keeps elements that are valid object ids" do
list = ["https://lain.com/users/lain", "invalid"]
assert {:error, "invalid"} == Recipients.cast(list)
assert {:ok, ["https://lain.com/users/lain"]} == Recipients.cast(list)
end
test "it works with a list" do
......
......@@ -624,6 +624,8 @@ test "it sends notifications to mentioned users in new messages" do
"actor" => user.ap_id,
"object" => %{
"type" => "Note",
"id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(),
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"content" => "message with a Mention tag, but no explicit tagging",
"tag" => [
%{
......@@ -655,6 +657,9 @@ test "it does not send notifications to users who are only cc in new messages" d
"actor" => user.ap_id,
"object" => %{
"type" => "Note",
"id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(),
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"cc" => [other_user.ap_id],
"content" => "hi everyone",
"attributedTo" => user.ap_id
}
......@@ -951,6 +956,7 @@ test "notifications are deleted if a remote user is deleted" do
"cc" => [],
"object" => %{
"type" => "Note",
"id" => remote_user.ap_id <> "/objects/test",
"content" => "Hello!",
"tag" => [
%{
......
......@@ -539,7 +539,7 @@ test "it inserts an incoming activity into the database" <>
File.read!("test/fixtures/mastodon-post-activity.json")
|> Jason.decode!()
|> Map.put("actor", user.ap_id)
|> put_in(["object", "attridbutedTo"], user.ap_id)
|> put_in(["object", "attributedTo"], user.ap_id)
conn =
conn
......@@ -820,29 +820,34 @@ test "it clears `unreachable` federation status of the sender", %{conn: conn, da
assert Instances.reachable?(sender_host)
end
@tag capture_log: true
test "it removes all follower collections but actor's", %{conn: conn} do
[actor, recipient] = insert_pair(:user)
data =
File.read!("test/fixtures/activitypub-client-post-activity.json")
|> Jason.decode!()
to = [
recipient.ap_id,
recipient.follower_address,
"https://www.w3.org/ns/activitystreams#Public"
]
object = Map.put(data["object"], "attributedTo", actor.ap_id)
cc = [recipient.follower_address, actor.follower_address]
data =
data
|> Map.put("id", Utils.generate_object_id())
|> Map.put("actor", actor.ap_id)
|> Map.put("object", object)
|> Map.put("cc", [
recipient.follower_address,
actor.follower_address
])
|> Map.put("to", [
recipient.ap_id,
recipient.follower_address,
"https://www.w3.org/ns/activitystreams#Public"
])
data = %{
"@context" => ["https://www.w3.org/ns/activitystreams"],
"type" => "Create",
"id" => Utils.generate_activity_id(),
"to" => to,
"cc" => cc,
"actor" => actor.ap_id,
"object" => %{
"type" => "Note",
"to" => to,
"cc" => cc,
"content" => "It's a note",
"attributedTo" => actor.ap_id,
"id" => Utils.generate_object_id()
}
}
conn
|> assign(:valid_signature, true)
......@@ -852,7 +857,7 @@ test "it removes all follower collections but actor's", %{conn: conn} do
ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
activity = Activity.get_by_ap_id(data["id"])
assert activity = Activity.get_by_ap_id(data["id"])
assert activity.id
assert actor.follower_address in activity.recipients
......
......@@ -14,7 +14,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
import Mock
import Pleroma.Factory
import ExUnit.CaptureLog
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
......@@ -147,9 +146,7 @@ test "it does not crash if the object in inReplyTo can't be fetched" do
data
|> Map.put("object", object)
assert capture_log(fn ->
{:ok, _returned_activity} = Transmogrifier.handle_incoming(data)
end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil"
assert {:ok, _returned_activity} = Transmogrifier.handle_incoming(data)
end
test "it does not work for deactivated users" do
......@@ -221,8 +218,25 @@ test "it works for incoming notices with hashtags" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"], fetch: false)
assert Enum.at(Object.tags(object), 2) == "moo"
assert Object.hashtags(object) == ["moo"]
assert match?(
%{
"href" => "http://localtesting.pleroma.lol/users/lain",
"name" => "@lain@localtesting.pleroma.lol",
"type" => "Mention"
},
Enum.at(object.data["tag"], 0)
)
assert match?(
%{
"href" => "http://mastodon.example.org/tags/moo",
"name" => "#moo",
"type" => "Hashtag"
},
Enum.at(object.data["tag"], 1)
)
assert "moo" == Enum.at(object.data["tag"], 2)
end
test "it works for incoming notices with contentMap" do
......@@ -276,13 +290,11 @@ test "it ensures that address fields become lists" do
File.read!("test/fixtures/mastodon-post-activity.json")
|> Jason.decode!()
|> Map.put("actor", user.ap_id)
|> Map.put("to", nil)
|> Map.put("cc", nil)
object =
data["object"]
|> Map.put("attributedTo", user.ap_id)
|> Map.put("to", nil)
|> Map.put("cc", nil)
|> Map.put("id", user.ap_id <> "/activities/12345678")
......@@ -290,8 +302,7 @@ test "it ensures that address fields become lists" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
assert !is_nil(data["to"])
assert !is_nil(data["cc"])
refute is_nil(data["cc"])
end
test "it strips internal likes" do
......@@ -330,70 +341,46 @@ test "it strips internal reactions" do
end
test "it correctly processes messages with non-array to field" do
user = insert(:user)
data =
File.read!("test/fixtures/mastodon-post-activity.json")
|> Poison.decode!()
|> Map.put("to", "https://www.w3.org/ns/activitystreams#Public")
|> put_in(["object", "to"], "https://www.w3.org/ns/activitystreams#Public")
message = %{
"@context" => "https://www.w3.org/ns/activitystreams",
"to" => "https://www.w3.org/ns/activitystreams#Public",
"type" => "Create",
"object" => %{
"content" => "blah blah blah",
"type" => "Note",
"attributedTo" => user.ap_id,
"inReplyTo" => nil
},
"actor" => user.ap_id
}
assert {:ok, activity} = Transmogrifier.handle_incoming(data)
assert {:ok, activity} = Transmogrifier.handle_incoming(message)
assert [
"http://mastodon.example.org/users/admin/followers",
"http://localtesting.pleroma.lol/users/lain"
] == activity.data["cc"]
assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
end
test "it correctly processes messages with non-array cc field" do
user = insert(:user)
message = %{
"@context" => "https://www.w3.org/ns/activitystreams",
"to" => user.follower_address,
"cc" => "https://www.w3.org/ns/activitystreams#Public",
"type" => "Create",
"object" => %{
"content" => "blah blah blah",
"type" => "Note",
"attributedTo" => user.ap_id,
"inReplyTo" => nil
},
"actor" => user.ap_id
}
data =
File.read!("test/fixtures/mastodon-post-activity.json")
|> Poison.decode!()
|> Map.put("cc", "http://mastodon.example.org/users/admin/followers")
|> put_in(["object", "cc"], "http://mastodon.example.org/users/admin/followers")
assert {:ok, activity} = Transmogrifier.handle_incoming(message)
assert {:ok, activity} = Transmogrifier.handle_incoming(data)
assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"]
assert [user.follower_address] == activity.data["to"]
assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"]
assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
end
test "it correctly processes messages with weirdness in address fields" do
user = insert(:user)
message = %{
"@context" => "https://www.w3.org/ns/activitystreams",
"to" => [nil, user.follower_address],
"cc" => ["https://www.w3.org/ns/activitystreams#Public", ["¿"]],
"type" => "Create",
"object" => %{
"content" => "…",
"type" => "Note",
"attributedTo" => user.ap_id,
"inReplyTo" => nil
},
"actor" => user.ap_id
}
data =
File.read!("test/fixtures/mastodon-post-activity.json")
|> Poison.decode!()
|> Map.put("cc", ["http://mastodon.example.org/users/admin/followers", ["¿"]])
|> put_in(["object", "cc"], ["http://mastodon.example.org/users/admin/followers", ["¿"]])
assert {:ok, activity} = Transmogrifier.handle_incoming(message)
assert {:ok, activity} = Transmogrifier.handle_incoming(data)