Merge branch 'develop' into 'feature/relay'

# Conflicts:
#   lib/pleroma/web/activity_pub/utils.ex

config/config.exs

@@ -16,6 +16,8 @@
 
 config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
 
+config :pleroma, :uri_schemes, additionnal_schemes: []
+
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
   url: [host: "localhost"],
@@ -71,11 +73,8 @@
   redirect_root_no_login: "/main/all",
   redirect_root_login: "/main/friends",
   show_instance_panel: true,
-  show_who_to_follow_panel: false,
-  who_to_follow_provider:
-    "https://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-osa-api.cgi?{{host}}+{{user}}",
-  who_to_follow_link: "https://vinayaka.distsn.org/?{{host}}+{{user}}",
-  scope_options_enabled: false
+  scope_options_enabled: false,
+  collapse_message_with_subject: false
 
 config :pleroma, :activitypub,
   accept_blocks: true,
@@ -112,6 +111,13 @@
   ip: {0, 0, 0, 0},
   port: 9999
 
+config :pleroma, :suggestions,
+  enabled: false,
+  third_party_engine:
+    "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}",
+  timeout: 300_000,
+  web: "https://vinayaka.distsn.org/?{{host}}+{{user}}"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

installation/caddyfile-pleroma.example

 social.domain.tld  {
-  tls user@domain.tld
+  log /var/log/caddy/pleroma_access.log
+  errors /var/log/caddy/pleroma_error.log
 
-  log /var/log/caddy/pleroma.log
+  gzip
 
+  proxy / localhost:4000 {
+    websocket
+    transparent
+  }
+
+  tls user@domain.tld {
+    # Remove the rest of the lines in here, if you want to support older devices
+    key_type p256
+    ciphers ECDHE-ECDSA-WITH-CHACHA20-POLY1305 ECDHE-RSA-WITH-CHACHA20-POLY1305 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256
+  }
+
+  header / {
+    X-XSS-Protection "1; mode=block"
+    X-Frame-Options "DENY"
+    X-Content-Type-Options "nosniff"
+    Referrer-Policy "same-origin"
+    Strict-Transport-Security "max-age=31536000; includeSubDomains;"
+    Expect-CT "enforce, max-age=2592000"
+  }
+
+  # If you do not want remote frontends to be able to access your Pleroma backend server, remove these lines.
+  # If you want to allow all origins access, remove the origin lines.
+  # To use this directive, you need the http.cors plugin for Caddy.
   cors / {
     origin https://halcyon.domain.tld
     origin https://pinafore.domain.tld
@@ -10,9 +34,13 @@ social.domain.tld  {
     allowed_headers Authorization,Content-Type,Idempotency-Key
     exposed_headers Link,X-RateLimit-Reset,X-RateLimit-Limit,X-RateLimit-Remaining,X-Request-Id
   }
+  # Stop removing lines here.
 
-  proxy / localhost:4000 {
-    websocket
-    transparent
+  # If you do not want to use the mediaproxy function, remove these lines.
+  # To use this directive, you need the http.cache plugin for Caddy.
+  cache {
+    match_path /proxy
+    default_max_age 720m
   }
+  # Stop removing lines here.
 }

installation/init.d/pleroma

+#!/sbin/openrc-run
+
+# Requires OpenRC >= 0.35
+directory=~pleroma/pleroma
+
+command=/usr/bin/mix
+command_args="phx.server"
+command_user=pleroma:pleroma
+command_background=1
+
+export PORT=4000
+export MIX_ENV=prod
+
+# Ask process to terminate within 30 seconds, otherwise kill it
+retry="SIGTERM/30 SIGKILL/5"
+
+pidfile="/var/run/pleroma.pid"
+
+depend() {
+    need nginx postgresql
+}
\ No newline at end of file

lib/mix/tasks/generate_invite_token.ex

+defmodule Mix.Tasks.GenerateInviteToken do
+  use Mix.Task
+
+  @shortdoc "Generate invite token for user"
+  def run([]) do
+    Mix.Task.run("app.start")
+
+    with {:ok, token} <- Pleroma.UserInviteToken.create_token() do
+      IO.puts("Generated user invite token")
+
+      IO.puts(
+        "Url: #{
+          Pleroma.Web.Router.Helpers.redirect_url(
+            Pleroma.Web.Endpoint,
+            :registration_page,
+            token.token
+          )
+        }"
+      )
+    else
+      _ ->
+        IO.puts("Error creating token")
+    end
+  end
+end

lib/pleroma/formatter.ex

@@ -16,7 +16,7 @@ def parse_tags(text, data \\ %{}) do
   def parse_mentions(text) do
     # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
     regex =
-      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
+      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
 
     Regex.scan(regex, text)
     |> List.flatten()
@@ -165,8 +165,29 @@ def get_custom_emoji() do
     @emoji
   end
 
-  @link_regex ~r/https?:\/\/[\w\.\/?=\-#\+%&@~'\(\):]+[\w\/]/u
+  @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
+
+  # IANA got a list https://www.iana.org/assignments/uri-schemes/ but
+  # Stuff like ipfs isn’t in it
+  # There is very niche stuff
+  @uri_schemes [
+    "https://",
+    "http://",
+    "dat://",
+    "dweb://",
+    "gopher://",
+    "ipfs://",
+    "ipns://",
+    "irc:",
+    "ircs:",
+    "magnet:",
+    "mailto:",
+    "mumble:",
+    "ssb://",
+    "xmpp:"
+  ]
 
+  # TODO: make it use something other than @link_regex
   def html_escape(text) do
     Regex.split(@link_regex, text, include_captures: true)
     |> Enum.map_every(2, fn chunk ->
@@ -176,11 +197,18 @@ def html_escape(text) do
     |> Enum.join("")
   end
 
-  @doc "changes http:... links to html links"
+  @doc "changes scheme:... urls to html links"
   def add_links({subs, text}) do
+    additionnal_schemes =
+      Application.get_env(:pleroma, :uri_schemes, [])
+      |> Keyword.get(:additionnal_schemes, [])
+
     links =
-      Regex.scan(@link_regex, text)
-      |> Enum.map(fn [url] -> {Ecto.UUID.generate(), url} end)
+      text
+      |> String.split([" ", "\t", "<br>"])
+      |> Enum.filter(fn word -> String.starts_with?(word, @uri_schemes ++ additionnal_schemes) end)
+      |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end)
+      |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end)
       |> Enum.sort_by(fn {_, url} -> -String.length(url) end)
 
     uuid_text =
@@ -244,8 +272,8 @@ def add_hashtag_links({subs, text}, tags) do
 
     subs =
       subs ++
-        Enum.map(tags, fn {_, tag, uuid} ->
-          url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>##{tag}</a>"
+        Enum.map(tags, fn {tag_text, tag, uuid} ->
+          url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{tag_text}</a>"
           {uuid, url}
         end)
 

lib/pleroma/gopher/server.ex

@@ -54,7 +54,7 @@ def info(text) do
 
     String.split(text, "\r")
     |> Enum.map(fn text ->
-      "i#{text}\tfake\(NULL)\t0\r\n"
+      "i#{text}\tfake\t(NULL)\t0\r\n"
     end)
     |> Enum.join("")
   end
@@ -77,14 +77,14 @@ def render_activities(activities) do
 
       link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <>
         info("#{like_count} likes, #{announcement_count} repeats") <>
-        "\r\n" <>
+        "i\tfake\t(NULL)\t0\r\n" <>
         info(
           HtmlSanitizeEx.strip_tags(
             String.replace(activity.data["object"]["content"], "<br>", "\r")
           )
         )
     end)
-    |> Enum.join("\r\n")
+    |> Enum.join("i\tfake\t(NULL)\t0\r\n")
   end
 
   def response("") do

lib/pleroma/http/http.ex

 defmodule Pleroma.HTTP do
-  use HTTPoison.Base
+  require HTTPoison
+
+  def request(method, url, body \\ "", headers \\ [], options \\ []) do
+    options =
+      process_request_options(options)
+      |> process_sni_options(url)
+
+    HTTPoison.request(method, url, body, headers, options)
+  end
+
+  defp process_sni_options(options, url) do
+    uri = URI.parse(url)
+    host = uri.host |> to_charlist()
+
+    case uri.scheme do
+      "https" -> options ++ [ssl: [server_name_indication: host]]
+      _ -> options
+    end
+  end
 
   def process_request_options(options) do
     config = Application.get_env(:pleroma, :http, [])
@@ -10,4 +28,9 @@ def process_request_options(options) do
       _ -> options ++ [proxy: proxy]
     end
   end
+
+  def get(url, headers \\ [], options \\ []), do: request(:get, url, "", headers, options)
+
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
 end

lib/pleroma/upload.ex

@@ -124,20 +124,20 @@ defp get_name(file, uuid, type, should_dedupe) do
     if should_dedupe do
       create_name(uuid, List.last(String.split(file.filename, ".")), type)
     else
-      unless String.contains?(file.filename, ".") do
-        case type do
-          "image/png" -> file.filename <> ".png"
-          "image/jpeg" -> file.filename <> ".jpg"
-          "image/gif" -> file.filename <> ".gif"
-          "video/webm" -> file.filename <> ".webm"
-          "video/mp4" -> file.filename <> ".mp4"
-          "audio/mpeg" -> file.filename <> ".mp3"
-          "audio/ogg" -> file.filename <> ".ogg"
-          "audio/wav" -> file.filename <> ".wav"
-          _ -> file.filename
+      parts = String.split(file.filename, ".")
+
+      new_filename =
+        if length(parts) > 1 do
+          Enum.drop(parts, -1) |> Enum.join(".")
+        else
+          Enum.join(parts)
         end
-      else
-        file.filename
+
+      case type do
+        "application/octet-stream" -> file.filename
+        "audio/mpeg" -> new_filename <> ".mp3"
+        "image/jpeg" -> new_filename <> ".jpg"
+        _ -> Enum.join([new_filename, String.split(type, "/") |> List.last()], ".")
       end
     end
   end

lib/pleroma/user.ex

@@ -398,6 +398,7 @@ def get_follow_requests(%User{} = user) do
       Enum.map(reqs, fn req -> req.actor end)
       |> Enum.uniq()
       |> Enum.map(fn ap_id -> get_by_ap_id(ap_id) end)
+      |> Enum.filter(fn u -> !following?(u, user) end)
 
     {:ok, users}
   end

lib/pleroma/user_invite_token.ex

+defmodule Pleroma.UserInviteToken do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias Pleroma.{User, UserInviteToken, Repo}
+
+  schema "user_invite_tokens" do
+    field(:token, :string)
+    field(:used, :boolean, default: false)
+
+    timestamps()
+  end
+
+  def create_token do
+    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64()
+
+    token = %UserInviteToken{
+      used: false,
+      token: token
+    }
+
+    Repo.insert(token)
+  end
+
+  def used_changeset(struct) do
+    struct
+    |> cast(%{}, [])
+    |> put_change(:used, true)
+  end
+
+  def mark_as_used(token) do
+    with %{used: false} = token <- Repo.get_by(UserInviteToken, %{token: token}),
+         {:ok, token} <- Repo.update(used_changeset(token)) do
+      {:ok, token}
+    else
+      _e -> {:error, token}
+    end
+  end
+end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -576,7 +576,7 @@ def user_data_from_user_object(data) do
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
     with {:ok, %{status_code: 200, body: body}} <-
-           @httpoison.get(ap_id, Accept: "application/activity+json"),
+           @httpoison.get(ap_id, [Accept: "application/activity+json"], follow_redirect: true),
          {:ok, data} <- Jason.decode(body) do
       user_data_from_user_object(data)
     else

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -18,12 +18,16 @@ def get_actor(%{"actor" => actor}) when is_binary(actor) do
   end
 
   def get_actor(%{"actor" => actor}) when is_list(actor) do
-    Enum.at(actor, 0)
+    if is_binary(Enum.at(actor, 0)) do
+      Enum.at(actor, 0)
+    else
+      Enum.find(actor, fn %{"type" => type} -> type == "Person" end)
+      |> Map.get("id")
+    end
   end
 
-  def get_actor(%{"actor" => actor_list}) do
-    Enum.find(actor_list, fn %{"type" => type} -> type == "Person" end)
-    |> Map.get("id")
+  def get_actor(%{"actor" => actor}) when is_map(actor) do
+    actor["id"]
   end
 
   @doc """
@@ -38,6 +42,25 @@ def fix_object(object) do
     |> fix_emoji
     |> fix_tag
     |> fix_content_map
+    |> fix_likes
+    |> fix_addressing
+  end
+
+  def fix_addressing_list(map, field) do
+    if is_binary(map[field]) do
+      map
+      |> Map.put(field, [map[field]])
+    else
+      map
+    end
+  end
+
+  def fix_addressing(map) do
+    map
+    |> fix_addressing_list("to")
+    |> fix_addressing_list("cc")
+    |> fix_addressing_list("bto")
+    |> fix_addressing_list("bcc")
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -45,6 +68,20 @@ def fix_actor(%{"attributedTo" => actor} = object) do
     |> Map.put("actor", get_actor(%{"actor" => actor}))
   end
 
+  def fix_likes(%{"likes" => likes} = object)
+      when is_bitstring(likes) do
+    # Check for standardisation
+    # This is what Peertube does
+    # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+    object
+    |> Map.put("likes", [])
+    |> Map.put("like_count", 0)
+  end
+
+  def fix_likes(object) do
+    object
+  end
+
   def fix_in_reply_to(%{"inReplyTo" => in_reply_to_id} = object)
       when not is_nil(in_reply_to_id) do
     case ActivityPub.fetch_object_from_id(in_reply_to_id) do
@@ -72,8 +109,11 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to_id} = object)
   def fix_in_reply_to(object), do: object
 
   def fix_context(object) do
+    context = object["context"] || object["conversation"] || Utils.generate_context_id()
+
     object
-    |> Map.put("context", object["conversation"])
+    |> Map.put("context", context)
+    |> Map.put("conversation", context)
   end
 
   def fix_attachments(object) do
@@ -137,13 +177,22 @@ def fix_content_map(%{"contentMap" => content_map} = object) do
 
   def fix_content_map(object), do: object
 
+  # disallow objects with bogus IDs
+  def handle_incoming(%{"id" => nil}), do: :error
+  def handle_incoming(%{"id" => ""}), do: :error
+  # length of https:// = 8, should validate better, but good enough for now.
+  def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error
+
   # TODO: validate those with a Ecto scheme
   # - tags
   # - emoji
   def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data)
-      when objtype in ["Article", "Note"] do
+      when objtype in ["Article", "Note", "Video"] do
     actor = get_actor(data)
-    data = Map.put(data, "actor", actor)
+
+    data =
+      Map.put(data, "actor", actor)
+      |> fix_addressing
 
     with nil <- Activity.get_create_activity_by_object_ap_id(object["id"]),
          %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do

lib/pleroma/web/activity_pub/utils.ex

@@ -128,7 +128,7 @@ def lazy_put_object_defaults(map, activity \\ %{}) do
   Inserts a full object if it is contained in an activity.
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data})
-      when is_map(object_data) and type in ["Article", "Note"] do
+      when is_map(object_data) and type in ["Article", "Note", "Video"] do
     with {:ok, _} <- Object.create(object_data) do
       :ok
     end
@@ -204,13 +204,17 @@ def update_likes_in_object(likes, object) do
   end
 
   def add_like_to_object(%Activity{data: %{"actor" => actor}}, object) do
-    with likes <- [actor | object.data["likes"] || []] |> Enum.uniq() do
+    likes = if is_list(object.data["likes"]), do: object.data["likes"], else: []
+
+    with likes <- [actor | likes] |> Enum.uniq() do
       update_likes_in_object(likes, object)
     end
   end
 
   def remove_like_from_object(%Activity{data: %{"actor" => actor}}, object) do
-    with likes <- (object.data["likes"] || []) |> List.delete(actor) do
+    likes = if is_list(object.data["likes"]), do: object.data["likes"], else: []
+
+    with likes <- likes |> List.delete(actor) do
       update_likes_in_object(likes, object)
     end
   end
@@ -380,7 +384,10 @@ def add_announce_to_object(
         },
         object
       ) do
-    with announcements <- [actor | object.data["announcements"] || []] |> Enum.uniq() do
+    announcements =
+      if is_list(object.data["announcements"]), do: object.data["announcements"], else: []
+
+    with announcements <- [actor | announcements] |> Enum.uniq() do
       update_element_in_object("announcement", announcements, object)
     end
   end
@@ -388,7 +395,10 @@ def add_announce_to_object(
   def add_announce_to_object(_, object), do: {:ok, object}
 
   def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do
-    with announcements <- (object.data["announcements"] || []) |> List.delete(actor) do
+    announcements =
+      if is_list(object.data["announcements"]), do: object.data["announcements"], else: []
+
+    with announcements <- announcements |> List.delete(actor) do
       update_element_in_object("announcement", announcements, object)
     end
   end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -127,9 +127,6 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
     info = User.user_info(user)
 
     params = %{
-      "type" => ["Create", "Announce"],
-      "actor_id" => user.ap_id,
-      "whole_db" => true,
       "limit" => "10"
     }
 
@@ -140,10 +137,8 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
         params
       end
 
-    activities = ActivityPub.fetch_public_activities(params)
-    min_id = Enum.at(activities, 0).id
-
-    activities = Enum.reverse(activities)
+    activities = ActivityPub.fetch_user_activities(user, nil, params)
+    min_id = Enum.at(Enum.reverse(activities), 0).id
     max_id = Enum.at(activities, 0).id
 
     collection =

lib/pleroma/web/common_api/utils.ex

@@ -64,7 +64,6 @@ def to_for_user_and_mentions(_user, mentions, inReplyTo, "direct") do
 
   def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do
     status
-    |> String.replace("\r", "")
     |> format_input(mentions, tags)
     |> maybe_add_attachments(attachments, no_attachment_links)
   end
@@ -95,7 +94,7 @@ def add_attachments(text, attachments) do