Merge branch 'develop' into dtluna/pleroma-refactor/1

TODO.txt

-- Add cache for user fetching / representing. (mostly in TwitterAPI.activity_to_status)
-
 Unliking:
 
 - Add a proper undo activity, find out how to ignore those in twitter api.
+
+WEBSUB:
+
+- Add unsubscription
+- Add periodical renewal
+

config/config.exs

@@ -30,7 +30,8 @@
   "application/xrd+xml" => ["xrd+xml"]
 }
 
-config :pleroma, :websub_verifier, Pleroma.Web.Websub
+config :pleroma, :websub, Pleroma.Web.Websub
+config :pleroma, :ostatus, Pleroma.Web.OStatus
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.

config/test.exs

@@ -25,4 +25,5 @@
 # Reduce hash rounds for testing
 config :comeonin, :pbkdf2_rounds, 1
 
-config :pleroma, :websub_verifier, Pleroma.Web.WebsubMock
+config :pleroma, :websub, Pleroma.Web.WebsubMock
+config :pleroma, :ostatus, Pleroma.Web.OStatusMock

lib/pleroma/activity.ex

@@ -5,6 +5,7 @@ defmodule Pleroma.Activity do
 
   schema "activities" do
     field :data, :map
+    field :local, :boolean, default: true
 
     timestamps()
   end
@@ -18,4 +19,9 @@ def all_by_object_ap_id(ap_id) do
     Repo.all(from activity in Activity,
       where: fragment("? @> ?", activity.data, ^%{object: %{id: ap_id}}))
   end
+
+  def get_create_activity_by_object_ap_id(ap_id) do
+    Repo.one(from activity in Activity,
+      where: fragment("? @> ?", activity.data, ^%{type: "Create", object: %{id: ap_id}}))
+  end
 end

lib/pleroma/application.ex

@@ -15,9 +15,9 @@ def start(_type, _args) do
       # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
       # worker(Pleroma.Worker, [arg1, arg2, arg3]),
       worker(Cachex, [:user_cache, [
-                         default_ttl: 5000,
+                         default_ttl: 25000,
                          ttl_interval: 1000,
-                         limit: 500
+                         limit: 2500
                        ]])
     ]
 

lib/pleroma/object.ex

@@ -13,4 +13,24 @@ def get_by_ap_id(ap_id) do
     Repo.one(from object in Object,
       where: fragment("? @> ?", object.data, ^%{id: ap_id}))
   end
+
+  def get_cached_by_ap_id(ap_id) do
+    if Mix.env == :test do
+      get_by_ap_id(ap_id)
+    else
+      key = "object:#{ap_id}"
+      Cachex.get!(:user_cache, key, fallback: fn(_) ->
+        object = get_by_ap_id(ap_id)
+        if object do
+          {:commit, object}
+        else
+          {:ignore, object}
+        end
+      end)
+    end
+  end
+
+  def context_mapping(context) do
+    %Object{data: %{"id" => context}}
+  end
 end

lib/pleroma/user.ex

 defmodule Pleroma.User do
   use Ecto.Schema
+
   import Ecto.{Changeset, Query}
   alias Pleroma.{Repo, User, Object, Web}
   alias Comeonin.Pbkdf2
+  alias Pleroma.Web.OStatus
 
   schema "users" do
     field :bio, :string
@@ -15,6 +17,8 @@ defmodule Pleroma.User do
     field :following, {:array, :string}, default: []
     field :ap_id, :string
     field :avatar, :map
+    field :local, :boolean, default: true
+    field :info, :map, default: %{}
 
     timestamps()
   end
@@ -118,6 +122,27 @@ def get_cached_by_ap_id(ap_id) do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end)
+    Cachex.get!(:user_cache, key, fallback: fn(_) -> get_or_fetch_by_nickname(nickname) end)
+  end
+
+  def get_by_nickname(nickname) do
+    Repo.get_by(User, nickname: nickname)
+  end
+
+  def get_cached_user_info(user) do
+    key = "user_info:#{user.id}"
+    Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end)
+  end
+
+  def get_or_fetch_by_nickname(nickname) do
+    with %User{} = user <- get_by_nickname(nickname)  do
+      user
+    else _e ->
+      with [nick, domain] <- String.split(nickname, "@"),
+           {:ok, user} <- OStatus.make_user(nickname) do
+        user
+      else _e -> nil
+      end
+    end
   end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -3,7 +3,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Ecto.{Changeset, UUID}
   import Ecto.Query
 
-  def insert(map) when is_map(map) do
+  def insert(map, local \\ true) when is_map(map) do
     map = map
     |> Map.put_new_lazy("id", &generate_activity_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
@@ -16,7 +16,29 @@ def insert(map) when is_map(map) do
       map
     end
 
-    Repo.insert(%Activity{data: map})
+    Repo.insert(%Activity{data: map, local: local})
+  end
+
+  def create(to, actor, context, object, additional \\ %{}, published \\ nil, local \\ true) do
+    published = published || make_date()
+
+    activity = %{
+      "type" => "Create",
+      "to" => to |> Enum.uniq,
+      "actor" => actor.ap_id,
+      "object" => object,
+      "published" => published,
+      "context" => context
+    }
+    |> Map.merge(additional)
+
+    with {:ok, activity} <- insert(activity, local) do
+      if actor.local do
+        Pleroma.Web.Federator.enqueue(:publish, activity)
+       end
+
+      {:ok, activity}
+    end
   end
 
   def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
@@ -33,7 +55,8 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
           "type" => "Like",
           "actor" => ap_id,
           "object" => id,
-          "to" => [User.ap_followers(user), object.data["actor"]]
+          "to" => [User.ap_followers(user), object.data["actor"]],
+          "context" => object.data["context"]
         }
 
         {:ok, activity} = insert(data)
@@ -49,6 +72,10 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
 
         update_object_in_activities(object)
 
+        if user.local do
+          Pleroma.Web.Federator.enqueue(:publish, activity)
+        end
+
         {:ok, activity, object}
     end
   end
@@ -99,7 +126,7 @@ def generate_context_id do
   end
 
   def generate_object_id do
-    generate_id("objects")
+    Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, Ecto.UUID.generate)
   end
 
   def generate_id(type) do
@@ -127,6 +154,12 @@ def fetch_activities(recipients, opts \\ %{}) do
     query = from activity in query,
       where: activity.id > ^since_id
 
+    query = if opts["local_only"] do
+      from activity in query, where: activity.local == true
+    else
+      query
+    end
+
     query = if opts["max_id"] do
       from activity in query, where: activity.id < ^opts["max_id"]
     else
@@ -143,15 +176,16 @@ def fetch_activities(recipients, opts \\ %{}) do
     Enum.reverse(Repo.all(query))
   end
 
-  def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
+  def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do
     data = %{
       "type" => "Announce",
       "actor" => ap_id,
       "object" => id,
-      "to" => [User.ap_followers(user), object.data["actor"]]
+      "to" => [User.ap_followers(user), object.data["actor"]],
+      "context" => object.data["context"]
     }
 
-    {:ok, activity} = insert(data)
+    {:ok, activity} = insert(data, local)
 
     announcements = [ap_id | (object.data["announcements"] || [])] |> Enum.uniq
 
@@ -164,6 +198,10 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object)
 
     update_object_in_activities(object)
 
+    if user.local do
+      Pleroma.Web.Federator.enqueue(:publish, activity)
+    end
+
     {:ok, activity, object}
   end
 

lib/pleroma/web/federator/federator.ex

+defmodule Pleroma.Web.Federator do
+  alias Pleroma.User
+  alias Pleroma.Web.WebFinger
+  require Logger
+
+  @websub Application.get_env(:pleroma, :websub)
+
+  def handle(:publish, activity) do
+    Logger.debug("Running publish for #{activity.data["id"]}")
+    with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
+      Logger.debug("Sending #{activity.data["id"]} out via websub")
+      Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
+
+      {:ok, actor} = WebFinger.ensure_keys_present(actor)
+      Logger.debug("Sending #{activity.data["id"]} out via salmon")
+      Pleroma.Web.Salmon.publish(actor, activity)
+    end
+  end
+
+  def handle(:verify_websub, websub) do
+    Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})")
+    @websub.verify(websub)
+  end
+
+  def handle(type, payload) do
+    Logger.debug("Unknown task: #{type}")
+    {:error, "Don't know what do do with this"}
+  end
+
+  def enqueue(type, payload) do
+    # for now, just run immediately in a new process.
+    if Mix.env == :test do
+      handle(type, payload)
+    else
+      spawn(fn -> handle(type, payload) end)
+    end
+  end
+end

lib/pleroma/web/ostatus/activity_representer.ex

 defmodule Pleroma.Web.OStatus.ActivityRepresenter do
-  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do
+  alias Pleroma.{Activity, User}
+  alias Pleroma.Web.OStatus.UserRepresenter
+  require Logger
+
+  defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do
+    [{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}]
+  end
+
+  defp get_in_reply_to(_), do: []
+
+  defp get_mentions(to) do
+    Enum.map(to, fn (id) ->
+      cond do
+        # Special handling for the AP/Ostatus public collections
+        "https://www.w3.org/ns/activitystreams#Public" == id ->
+          {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []}
+        # Ostatus doesn't handle follower collections, ignore these.
+        Regex.match?(~r/^#{Pleroma.Web.base_url}.+followers$/, id) ->
+          []
+        true ->
+          {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []}
+      end
+    end)
+  end
+
+  def to_simple_form(activity, user, with_author \\ false)
+  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
     updated_at = activity.updated_at
@@ -12,16 +38,97 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user)
       {:link, [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], []}
     end)
 
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+    mentions = activity.data["to"] |> get_mentions
+
     [
       {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
-      {:id, h.(activity.data["object"]["id"])},
+      {:id, h.(activity.data["object"]["id"])}, # For notes, federate the object id.
       {:title, ['New note by #{user.nickname}']},
       {:content, [type: 'html'], h.(activity.data["object"]["content"])},
       {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)}
-    ] ++ attachments
+      {:updated, h.(updated_at)},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [type: ['application/atom+xml'], href: h.(activity.data["object"]["id"]), rel: 'self'], []}
+    ] ++ attachments ++ in_reply_to ++ author ++ mentions
+  end
+
+  def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
+    h = fn(str) -> [to_charlist(str)] end
+
+    updated_at = activity.updated_at
+    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.inserted_at
+    |> NaiveDateTime.to_iso8601
+
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+    mentions = activity.data["to"] |> get_mentions
+
+    [
+      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']},
+      {:id, h.(activity.data["id"])},
+      {:title, ['New favorite by #{user.nickname}']},
+      {:content, [type: 'html'], ['#{user.nickname} favorited something']},
+      {:published, h.(inserted_at)},
+      {:updated, h.(updated_at)},
+      {:"activity:object", [
+        {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
+        {:id, h.(activity.data["object"])}, # For notes, federate the object id.
+      ]},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
+      {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}
+    ] ++ author ++ mentions
+  end
+
+  def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
+    h = fn(str) -> [to_charlist(str)] end
+
+    updated_at = activity.updated_at
+    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.inserted_at
+    |> NaiveDateTime.to_iso8601
+
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+
+    retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
+
+    retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
+
+    mentions = activity.data["to"] |> get_mentions
+    [
+      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
+      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
+      {:id, h.(activity.data["id"])},
+      {:title, ['#{user.nickname} repeated a notice']},
+      {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']},
+      {:published, h.(inserted_at)},
+      {:updated, h.(updated_at)},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
+      {:"activity:object", retweeted_xml}
+    ] ++ mentions ++ author
+  end
+
+  def wrap_with_entry(simple_form) do
+    [{
+      :entry, [
+        xmlns: 'http://www.w3.org/2005/Atom',
+        "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
+        "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
+        "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
+        "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
+      ], simple_form
+    }]
   end
 
-  def to_simple_form(_, _), do: nil
+  def to_simple_form(_, _, _), do: nil
 end

lib/pleroma/web/ostatus/feed_representer.ex

@@ -17,14 +17,17 @@ def to_simple_form(user, activities, users) do
     [{
       :feed, [
         xmlns: 'http://www.w3.org/2005/Atom',
+        "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
         "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-        "xmlns:poco": 'http://portablecontacts.net/spec/1.0'
+        "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
+        "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
       ], [
         {:id, h.(OStatus.feed_path(user))},
         {:title, ['#{user.nickname}\'s timeline']},
         {:updated, h.(most_recent_update)},
         {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
-        {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []},
+        {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
+        {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []},
         {:author, UserRepresenter.to_simple_form(user)},
       ] ++ entries
     }]

lib/pleroma/web/ostatus/ostatus.ex

 defmodule Pleroma.Web.OStatus do
-  alias Pleroma.Web
+  import Ecto.Query
+  import Pleroma.Web.XML
+  require Logger
+
+  alias Pleroma.{Repo, User, Web, Object}
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.{WebFinger, Websub}
 
   def feed_path(user) do
     "#{user.ap_id}/feed.atom"
@@ -9,6 +15,199 @@ def pubsub_path(user) do
     "#{Web.base_url}/push/hub/#{user.nickname}"
   end
 
-  def user_path(user) do
+  def salmon_path(user) do
+    "#{user.ap_id}/salmon"
+  end
+
+  def handle_incoming(xml_string) do
+    doc = parse_document(xml_string)
+    entries = :xmerl_xpath.string('//entry', doc)
+
+    activities = Enum.map(entries, fn (entry) ->
+      {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry)
+      {:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry)
+
+      case verb do
+        'http://activitystrea.ms/schema/1.0/share' ->
+          with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity]
+        _ ->
+          case object_type do
+            'http://activitystrea.ms/schema/1.0/note' ->
+              with {:ok, activity} <- handle_note(entry, doc), do: activity
+            'http://activitystrea.ms/schema/1.0/comment' ->
+              with {:ok, activity} <- handle_note(entry, doc), do: activity
+            _ ->
+              Logger.error("Couldn't parse incoming document")
+              nil
+          end
+      end
+    end)
+    {:ok, activities}
+  end
+
+  def make_share(entry, doc, retweeted_activity) do
+    with {:ok, actor} <- find_make_or_update_user(doc),
+         %Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]),
+         {:ok, activity, object} = ActivityPub.announce(actor, object, false) do
+      {:ok, activity}
+    end
+  end
+
+  def handle_share(entry, doc) do
+    with [object] <- :xmerl_xpath.string('/entry/activity:object', entry),
+         {:ok, retweeted_activity} <-  handle_note(object, object),
+         {:ok, activity} <- make_share(entry, doc, retweeted_activity) do
+      {:ok, activity, retweeted_activity}
+    else
+      e -> {:error, e}
+    end
+  end
+
+  def get_attachments(entry) do
+    :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry)
+    |> Enum.map(fn (enclosure) ->
+      with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure),
+           type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do
+        %{
+          "type" => "Attachment",
+          "url" => [%{
+                       "type" => "Link",
+                       "mediaType" => type,
+                       "href" => href
+                    }]
+        }
+      end
+    end)
+    |> Enum.filter(&(&1))
+  end
+
+  def handle_note(entry, doc \\ nil) do
+    content_html = string_from_xpath("//content[1]", entry)
+
+    [author] = :xmerl_xpath.string('//author[1]', doc)
+    {:ok, actor} = find_make_or_update_user(author)
+    inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry)
+
+    context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim
+
+    attachments = get_attachments(entry)
+
+    context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do
+                context
+              else _e ->
+                if String.length(context) > 0 do
+                  context
+                else
+                  ActivityPub.generate_context_id
+                end
+              end
+
+    to = [
+      "https://www.w3.org/ns/activitystreams#Public"
+    ]
+
+    mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry)
+    |> Enum.map(fn(person) -> string_from_xpath("@href", person) end)
+
+    to = to ++ mentions
+
+    date = string_from_xpath("//published", entry)
+    id = string_from_xpath("//id", entry)
+
+    object = %{
+      "id" => id,
+      "type" => "Note",
+      "to" => to,
+      "content" => content_html,
+      "published" => date,
+      "context" => context,
+      "actor" => actor.ap_id,
+      "attachment" => attachments
+    }
+
+    object = if inReplyTo do
+      Map.put(object, "inReplyTo", inReplyTo)
+    else
+      object
+    end
+
+    # TODO: Bail out sooner and use transaction.
+    if Object.get_by_ap_id(id) do
+      {:error, "duplicate activity"}
+    else
+      ActivityPub.create(to, actor, context, object, %{}, date, false)
+    end
+  end
+
+  def find_make_or_update_user(doc) do
+    uri = string_from_xpath("//author/uri[1]", doc)
+    with {:ok, user} <- find_or_make_user(uri) do
+      avatar = make_avatar_object(doc)
+      if user.avatar != avatar do
+        change = Ecto.Changeset.change(user, %{avatar: avatar})
+        Repo.update(change)
+      else