Commit c48c381e authored by lain's avatar lain

Merge branch 'develop' into dtluna/pleroma-refactor/1

parents 6cf7c132 c85998ab
- Add cache for user fetching / representing. (mostly in TwitterAPI.activity_to_status)
Unliking:
- Add a proper undo activity, find out how to ignore those in twitter api.
WEBSUB:
- Add unsubscription
- Add periodical renewal
......@@ -30,7 +30,8 @@
"application/xrd+xml" => ["xrd+xml"]
}
config :pleroma, :websub_verifier, Pleroma.Web.Websub
config :pleroma, :websub, Pleroma.Web.Websub
config :pleroma, :ostatus, Pleroma.Web.OStatus
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
......
......@@ -25,4 +25,5 @@
# Reduce hash rounds for testing
config :comeonin, :pbkdf2_rounds, 1
config :pleroma, :websub_verifier, Pleroma.Web.WebsubMock
config :pleroma, :websub, Pleroma.Web.WebsubMock
config :pleroma, :ostatus, Pleroma.Web.OStatusMock
......@@ -5,6 +5,7 @@ defmodule Pleroma.Activity do
schema "activities" do
field :data, :map
field :local, :boolean, default: true
timestamps()
end
......@@ -18,4 +19,9 @@ def all_by_object_ap_id(ap_id) do
Repo.all(from activity in Activity,
where: fragment("? @> ?", activity.data, ^%{object: %{id: ap_id}}))
end
def get_create_activity_by_object_ap_id(ap_id) do
Repo.one(from activity in Activity,
where: fragment("? @> ?", activity.data, ^%{type: "Create", object: %{id: ap_id}}))
end
end
......@@ -15,9 +15,9 @@ def start(_type, _args) do
# Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
# worker(Pleroma.Worker, [arg1, arg2, arg3]),
worker(Cachex, [:user_cache, [
default_ttl: 5000,
default_ttl: 25000,
ttl_interval: 1000,
limit: 500
limit: 2500
]])
]
......
......@@ -13,4 +13,24 @@ def get_by_ap_id(ap_id) do
Repo.one(from object in Object,
where: fragment("? @> ?", object.data, ^%{id: ap_id}))
end
def get_cached_by_ap_id(ap_id) do
if Mix.env == :test do
get_by_ap_id(ap_id)
else
key = "object:#{ap_id}"
Cachex.get!(:user_cache, key, fallback: fn(_) ->
object = get_by_ap_id(ap_id)
if object do
{:commit, object}
else
{:ignore, object}
end
end)
end
end
def context_mapping(context) do
%Object{data: %{"id" => context}}
end
end
defmodule Pleroma.User do
use Ecto.Schema
import Ecto.{Changeset, Query}
alias Pleroma.{Repo, User, Object, Web}
alias Comeonin.Pbkdf2
alias Pleroma.Web.OStatus
schema "users" do
field :bio, :string
......@@ -15,6 +17,8 @@ defmodule Pleroma.User do
field :following, {:array, :string}, default: []
field :ap_id, :string
field :avatar, :map
field :local, :boolean, default: true
field :info, :map, default: %{}
timestamps()
end
......@@ -118,6 +122,27 @@ def get_cached_by_ap_id(ap_id) do
def get_cached_by_nickname(nickname) do
key = "nickname:#{nickname}"
Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end)
Cachex.get!(:user_cache, key, fallback: fn(_) -> get_or_fetch_by_nickname(nickname) end)
end
def get_by_nickname(nickname) do
Repo.get_by(User, nickname: nickname)
end
def get_cached_user_info(user) do
key = "user_info:#{user.id}"
Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end)
end
def get_or_fetch_by_nickname(nickname) do
with %User{} = user <- get_by_nickname(nickname) do
user
else _e ->
with [nick, domain] <- String.split(nickname, "@"),
{:ok, user} <- OStatus.make_user(nickname) do
user
else _e -> nil
end
end
end
end
......@@ -3,7 +3,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
alias Ecto.{Changeset, UUID}
import Ecto.Query
def insert(map) when is_map(map) do
def insert(map, local \\ true) when is_map(map) do
map = map
|> Map.put_new_lazy("id", &generate_activity_id/0)
|> Map.put_new_lazy("published", &make_date/0)
......@@ -16,7 +16,29 @@ def insert(map) when is_map(map) do
map
end
Repo.insert(%Activity{data: map})
Repo.insert(%Activity{data: map, local: local})
end
def create(to, actor, context, object, additional \\ %{}, published \\ nil, local \\ true) do
published = published || make_date()
activity = %{
"type" => "Create",
"to" => to |> Enum.uniq,
"actor" => actor.ap_id,
"object" => object,
"published" => published,
"context" => context
}
|> Map.merge(additional)
with {:ok, activity} <- insert(activity, local) do
if actor.local do
Pleroma.Web.Federator.enqueue(:publish, activity)
end
{:ok, activity}
end
end
def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
......@@ -33,7 +55,8 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
"type" => "Like",
"actor" => ap_id,
"object" => id,
"to" => [User.ap_followers(user), object.data["actor"]]
"to" => [User.ap_followers(user), object.data["actor"]],
"context" => object.data["context"]
}
{:ok, activity} = insert(data)
......@@ -49,6 +72,10 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
update_object_in_activities(object)
if user.local do
Pleroma.Web.Federator.enqueue(:publish, activity)
end
{:ok, activity, object}
end
end
......@@ -99,7 +126,7 @@ def generate_context_id do
end
def generate_object_id do
generate_id("objects")
Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, Ecto.UUID.generate)
end
def generate_id(type) do
......@@ -127,6 +154,12 @@ def fetch_activities(recipients, opts \\ %{}) do
query = from activity in query,
where: activity.id > ^since_id
query = if opts["local_only"] do
from activity in query, where: activity.local == true
else
query
end
query = if opts["max_id"] do
from activity in query, where: activity.id < ^opts["max_id"]
else
......@@ -143,15 +176,16 @@ def fetch_activities(recipients, opts \\ %{}) do
Enum.reverse(Repo.all(query))
end
def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do
data = %{
"type" => "Announce",
"actor" => ap_id,
"object" => id,
"to" => [User.ap_followers(user), object.data["actor"]]
"to" => [User.ap_followers(user), object.data["actor"]],
"context" => object.data["context"]
}
{:ok, activity} = insert(data)
{:ok, activity} = insert(data, local)
announcements = [ap_id | (object.data["announcements"] || [])] |> Enum.uniq
......@@ -164,6 +198,10 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object)
update_object_in_activities(object)
if user.local do
Pleroma.Web.Federator.enqueue(:publish, activity)
end
{:ok, activity, object}
end
......
defmodule Pleroma.Web.Federator do
alias Pleroma.User
alias Pleroma.Web.WebFinger
require Logger
@websub Application.get_env(:pleroma, :websub)
def handle(:publish, activity) do
Logger.debug("Running publish for #{activity.data["id"]}")
with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
Logger.debug("Sending #{activity.data["id"]} out via websub")
Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
{:ok, actor} = WebFinger.ensure_keys_present(actor)
Logger.debug("Sending #{activity.data["id"]} out via salmon")
Pleroma.Web.Salmon.publish(actor, activity)
end
end
def handle(:verify_websub, websub) do
Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})")
@websub.verify(websub)
end
def handle(type, payload) do
Logger.debug("Unknown task: #{type}")
{:error, "Don't know what do do with this"}
end
def enqueue(type, payload) do
# for now, just run immediately in a new process.
if Mix.env == :test do
handle(type, payload)
else
spawn(fn -> handle(type, payload) end)
end
end
end
defmodule Pleroma.Web.OStatus.ActivityRepresenter do
def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do
alias Pleroma.{Activity, User}
alias Pleroma.Web.OStatus.UserRepresenter
require Logger
defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do
[{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}]
end
defp get_in_reply_to(_), do: []
defp get_mentions(to) do
Enum.map(to, fn (id) ->
cond do
# Special handling for the AP/Ostatus public collections
"https://www.w3.org/ns/activitystreams#Public" == id ->
{:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []}
# Ostatus doesn't handle follower collections, ignore these.
Regex.match?(~r/^#{Pleroma.Web.base_url}.+followers$/, id) ->
[]
true ->
{:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []}
end
end)
end
def to_simple_form(activity, user, with_author \\ false)
def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
h = fn(str) -> [to_charlist(str)] end
updated_at = activity.updated_at
......@@ -12,16 +38,97 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user)
{:link, [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], []}
end)
in_reply_to = get_in_reply_to(activity.data)
author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
mentions = activity.data["to"] |> get_mentions
[
{:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
{:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
{:id, h.(activity.data["object"]["id"])},
{:id, h.(activity.data["object"]["id"])}, # For notes, federate the object id.
{:title, ['New note by #{user.nickname}']},
{:content, [type: 'html'], h.(activity.data["object"]["content"])},
{:published, h.(inserted_at)},
{:updated, h.(updated_at)}
] ++ attachments
{:updated, h.(updated_at)},
{:"ostatus:conversation", [], h.(activity.data["context"])},
{:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
{:link, [type: ['application/atom+xml'], href: h.(activity.data["object"]["id"]), rel: 'self'], []}
] ++ attachments ++ in_reply_to ++ author ++ mentions
end
def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
h = fn(str) -> [to_charlist(str)] end
updated_at = activity.updated_at
|> NaiveDateTime.to_iso8601
inserted_at = activity.inserted_at
|> NaiveDateTime.to_iso8601
in_reply_to = get_in_reply_to(activity.data)
author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
mentions = activity.data["to"] |> get_mentions
[
{:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']},
{:id, h.(activity.data["id"])},
{:title, ['New favorite by #{user.nickname}']},
{:content, [type: 'html'], ['#{user.nickname} favorited something']},
{:published, h.(inserted_at)},
{:updated, h.(updated_at)},
{:"activity:object", [
{:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
{:id, h.(activity.data["object"])}, # For notes, federate the object id.
]},
{:"ostatus:conversation", [], h.(activity.data["context"])},
{:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
{:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
{:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}
] ++ author ++ mentions
end
def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
h = fn(str) -> [to_charlist(str)] end
updated_at = activity.updated_at
|> NaiveDateTime.to_iso8601
inserted_at = activity.inserted_at
|> NaiveDateTime.to_iso8601
in_reply_to = get_in_reply_to(activity.data)
author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
mentions = activity.data["to"] |> get_mentions
[
{:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
{:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
{:id, h.(activity.data["id"])},
{:title, ['#{user.nickname} repeated a notice']},
{:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']},
{:published, h.(inserted_at)},
{:updated, h.(updated_at)},
{:"ostatus:conversation", [], h.(activity.data["context"])},
{:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
{:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
{:"activity:object", retweeted_xml}
] ++ mentions ++ author
end
def wrap_with_entry(simple_form) do
[{
:entry, [
xmlns: 'http://www.w3.org/2005/Atom',
"xmlns:thr": 'http://purl.org/syndication/thread/1.0',
"xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
"xmlns:poco": 'http://portablecontacts.net/spec/1.0',
"xmlns:ostatus": 'http://ostatus.org/schema/1.0'
], simple_form
}]
end
def to_simple_form(_, _), do: nil
def to_simple_form(_, _, _), do: nil
end
......@@ -17,14 +17,17 @@ def to_simple_form(user, activities, users) do
[{
:feed, [
xmlns: 'http://www.w3.org/2005/Atom',
"xmlns:thr": 'http://purl.org/syndication/thread/1.0',
"xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
"xmlns:poco": 'http://portablecontacts.net/spec/1.0'
"xmlns:poco": 'http://portablecontacts.net/spec/1.0',
"xmlns:ostatus": 'http://ostatus.org/schema/1.0'
], [
{:id, h.(OStatus.feed_path(user))},
{:title, ['#{user.nickname}\'s timeline']},
{:updated, h.(most_recent_update)},
{:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
{:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []},
{:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
{:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []},
{:author, UserRepresenter.to_simple_form(user)},
] ++ entries
}]
......
defmodule Pleroma.Web.OStatus do
alias Pleroma.Web
import Ecto.Query
import Pleroma.Web.XML
require Logger
alias Pleroma.{Repo, User, Web, Object}
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.{WebFinger, Websub}
def feed_path(user) do
"#{user.ap_id}/feed.atom"
......@@ -9,6 +15,199 @@ def pubsub_path(user) do
"#{Web.base_url}/push/hub/#{user.nickname}"
end
def user_path(user) do
def salmon_path(user) do
"#{user.ap_id}/salmon"
end
def handle_incoming(xml_string) do
doc = parse_document(xml_string)
entries = :xmerl_xpath.string('//entry', doc)
activities = Enum.map(entries, fn (entry) ->
{:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry)
{:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry)
case verb do
'http://activitystrea.ms/schema/1.0/share' ->
with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity]
_ ->
case object_type do
'http://activitystrea.ms/schema/1.0/note' ->
with {:ok, activity} <- handle_note(entry, doc), do: activity
'http://activitystrea.ms/schema/1.0/comment' ->
with {:ok, activity} <- handle_note(entry, doc), do: activity
_ ->
Logger.error("Couldn't parse incoming document")
nil
end
end
end)
{:ok, activities}
end
def make_share(entry, doc, retweeted_activity) do
with {:ok, actor} <- find_make_or_update_user(doc),
%Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]),
{:ok, activity, object} = ActivityPub.announce(actor, object, false) do
{:ok, activity}
end
end
def handle_share(entry, doc) do
with [object] <- :xmerl_xpath.string('/entry/activity:object', entry),
{:ok, retweeted_activity} <- handle_note(object, object),
{:ok, activity} <- make_share(entry, doc, retweeted_activity) do
{:ok, activity, retweeted_activity}
else
e -> {:error, e}
end
end
def get_attachments(entry) do
:xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry)
|> Enum.map(fn (enclosure) ->
with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure),
type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do
%{
"type" => "Attachment",
"url" => [%{
"type" => "Link",
"mediaType" => type,
"href" => href
}]
}
end
end)
|> Enum.filter(&(&1))
end
def handle_note(entry, doc \\ nil) do
content_html = string_from_xpath("//content[1]", entry)
[author] = :xmerl_xpath.string('//author[1]', doc)
{:ok, actor} = find_make_or_update_user(author)
inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry)
context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim
attachments = get_attachments(entry)
context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do
context
else _e ->
if String.length(context) > 0 do
context
else
ActivityPub.generate_context_id
end
end
to = [
"https://www.w3.org/ns/activitystreams#Public"
]
mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry)
|> Enum.map(fn(person) -> string_from_xpath("@href", person) end)
to = to ++ mentions
date = string_from_xpath("//published", entry)
id = string_from_xpath("//id", entry)
object = %{
"id" => id,
"type" => "Note",
"to" => to,
"content" => content_html,
"published" => date,
"context" => context,
"actor" => actor.ap_id,
"attachment" => attachments
}
object = if inReplyTo do
Map.put(object, "inReplyTo", inReplyTo)
else
object
end