Merge remote-tracking branch 'origin' into follower-hiding

config/config.exs

@@ -72,6 +72,7 @@
 config :pleroma, :websub, Pleroma.Web.Websub
 config :pleroma, :ostatus, Pleroma.Web.OStatus
 config :pleroma, :httpoison, Pleroma.HTTP
+config :tesla, adapter: Tesla.Adapter.Hackney
 
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http, proxy_url: nil

config/test.exs

@@ -25,7 +25,7 @@
 
 config :pleroma, :websub, Pleroma.Web.WebsubMock
 config :pleroma, :ostatus, Pleroma.Web.OStatusMock
-config :pleroma, :httpoison, HTTPoisonMock
+config :tesla, adapter: Tesla.Mock
 
 try do
   import_config "test.secret.exs"

lib/mix/tasks/make_moderator.ex

@@ -8,7 +8,7 @@ defmodule Mix.Tasks.SetModerator do
   """
 
   use Mix.Task
-  import Mix.Ecto
+  import Ecto.Changeset
   alias Pleroma.{Repo, User}
 
   def run([nickname | rest]) do
@@ -21,14 +21,15 @@ def run([nickname | rest]) do
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("is_moderator", !!moderator)
+      info_cng = User.Info.admin_api_update(user.info, %{is_moderator: !!moderator})
 
-      cng = User.info_changeset(user, %{info: info})
-      {:ok, user} = User.update_and_set_cache(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("Moderator status of #{nickname}: #{user.info["is_moderator"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Moderator status of #{nickname}: #{user.info.is_moderator}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/mix/tasks/sample_psql.eex

@@ -4,3 +4,4 @@ CREATE DATABASE pleroma_dev OWNER pleroma;
 --Extensions made by ecto.migrate that need superuser access
 CREATE EXTENSION IF NOT EXISTS citext;
 CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

lib/mix/tasks/set_admin.ex

 defmodule Mix.Tasks.SetAdmin do
   use Mix.Task
+  import Ecto.Changeset
   alias Pleroma.User
 
   @doc """
@@ -9,21 +10,22 @@ defmodule Mix.Tasks.SetAdmin do
   def run([nickname | rest]) do
     Application.ensure_all_started(:pleroma)
 
-    status =
+    admin =
       case rest do
-        [status] -> status == "true"
+        [admin] -> admin == "true"
         _ -> true
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("is_admin", !!status)
+      info_cng = User.Info.admin_api_update(user.info, %{is_admin: !!admin})
 
-      cng = User.info_changeset(user, %{info: info})
-      {:ok, user} = User.update_and_set_cache(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("Admin status of #{nickname}: #{user.info["is_admin"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Admin status of #{nickname}: #{user.info.is_admin}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/mix/tasks/set_locked.ex

@@ -10,11 +10,11 @@ defmodule Mix.Tasks.SetLocked do
   """
 
   use Mix.Task
-  import Mix.Ecto
+  import Ecto.Changeset
   alias Pleroma.{Repo, User}
 
   def run([nickname | rest]) do
-    ensure_started(Repo, [])
+    Application.ensure_all_started(:pleroma)
 
     locked =
       case rest do
@@ -23,14 +23,15 @@ def run([nickname | rest]) do
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("locked", !!locked)
+      info_cng = User.Info.profile_update(user.info, %{locked: !!locked})
 
-      cng = User.info_changeset(user, %{info: info})
-      user = Repo.update!(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("locked status of #{nickname}: #{user.info["locked"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Locked status of #{nickname}: #{user.info.locked}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/pleroma/http/connection.ex

+defmodule Pleroma.HTTP.Connection do
+  @moduledoc """
+  Connection for http-requests.
+  """
+
+  @hackney_options [pool: :default]
+  @adapter Application.get_env(:tesla, :adapter)
+
+  @doc """
+  Configure a client connection
+
+  # Returns
+
+  Tesla.Env.client
+  """
+  @spec new(Keyword.t()) :: Tesla.Env.client()
+  def new(opts \\ []) do
+    Tesla.client([], {@adapter, hackney_options(opts)})
+  end
+
+  # fetch Hackney options
+  #
+  defp hackney_options(opts \\ []) do
+    options = Keyword.get(opts, :adapter, [])
+    @hackney_options ++ options
+  end
+end

lib/pleroma/http/http.ex

 defmodule Pleroma.HTTP do
-  require HTTPoison
+  @moduledoc """
 
+  """
+
+  alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.RequestBuilder, as: Builder
+
+  @doc """
+  Builds and perform http request.
+
+  # Arguments:
+  `method` - :get, :post, :put, :delete
+  `url`
+  `body`
+  `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
+  `options` - custom, per-request middleware or adapter options
+
+  # Returns:
+  `{:ok, %Tesla.Env{}}` or `{:error, error}`
+
+  """
   def request(method, url, body \\ "", headers \\ [], options \\ []) do
     options =
       process_request_options(options)
       |> process_sni_options(url)
 
-    HTTPoison.request(method, url, body, headers, options)
+    %{}
+    |> Builder.method(method)
+    |> Builder.headers(headers)
+    |> Builder.opts(options)
+    |> Builder.url(url)
+    |> Builder.add_param(:body, :body, body)
+    |> Enum.into([])
+    |> (&Tesla.request(Connection.new(), &1)).()
   end
 
+  defp process_sni_options(options, nil), do: options
+
   defp process_sni_options(options, url) do
     uri = URI.parse(url)
     host = uri.host |> to_charlist()
@@ -22,7 +50,7 @@ defp process_sni_options(options, url) do
   def process_request_options(options) do
     config = Application.get_env(:pleroma, :http, [])
     proxy = Keyword.get(config, :proxy_url, nil)
-    options = options ++ [hackney: [pool: :default]]
+    options = options ++ [adapter: [pool: :default]]
 
     case proxy do
       nil -> options
@@ -30,8 +58,19 @@ def process_request_options(options) do
     end
   end
 
-  def get(url, headers \\ [], options \\ []), do: request(:get, url, "", headers, options)
+  @doc """
+  Performs GET request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  def get(url, headers \\ [], options \\ []),
+    do: request(:get, url, "", headers, options)
+
+  @doc """
+  Performs POST request.
 
+  See `Pleroma.HTTP.request/5`
+  """
   def post(url, body, headers \\ [], options \\ []),
     do: request(:post, url, body, headers, options)
 end

lib/pleroma/http/request_builder.ex

+defmodule Pleroma.HTTP.RequestBuilder do
+  @moduledoc """
+  Helper functions for building Tesla requests
+  """
+
+  @doc """
+  Specify the request method when building a request
+
+  ## Parameters
+
+  - request (Map) - Collected request options
+  - m (atom) - Request method
+
+  ## Returns
+
+  Map
+  """
+  @spec method(map(), atom) :: map()
+  def method(request, m) do
+    Map.put_new(request, :method, m)
+  end
+
+  @doc """
+  Specify the request method when building a request
+
+  ## Parameters
+
+  - request (Map) - Collected request options
+  - u (String) - Request URL
+
+  ## Returns
+
+  Map
+  """
+  @spec url(map(), String.t()) :: map()
+  def url(request, u) do
+    Map.put_new(request, :url, u)
+  end
+
+  @doc """
+  Add headers to the request
+  """
+  @spec headers(map(), list(tuple)) :: map()
+  def headers(request, h) do
+    Map.put_new(request, :headers, h)
+  end
+
+  @doc """
+  Add custom, per-request middleware or adapter options to the request
+  """
+  @spec opts(map(), Keyword.t()) :: map()
+  def opts(request, options) do
+    Map.put_new(request, :opts, options)
+  end
+
+  @doc """
+  Add optional parameters to the request
+
+  ## Parameters
+
+  - request (Map) - Collected request options
+  - definitions (Map) - Map of parameter name to parameter location.
+  - options (KeywordList) - The provided optional parameters
+
+  ## Returns
+
+  Map
+  """
+  @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
+  def add_optional_params(request, _, []), do: request
+
+  def add_optional_params(request, definitions, [{key, value} | tail]) do
+    case definitions do
+      %{^key => location} ->
+        request
+        |> add_param(location, key, value)
+        |> add_optional_params(definitions, tail)
+
+      _ ->
+        add_optional_params(request, definitions, tail)
+    end
+  end
+
+  @doc """
+  Add optional parameters to the request
+
+  ## Parameters
+
+  - request (Map) - Collected request options
+  - location (atom) - Where to put the parameter
+  - key (atom) - The name of the parameter
+  - value (any) - The value of the parameter
+
+  ## Returns
+
+  Map
+  """
+  @spec add_param(map(), atom, atom, any()) :: map()
+  def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
+
+  def add_param(request, :body, key, value) do
+    request
+    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.update!(
+      :body,
+      &Tesla.Multipart.add_field(&1, key, Poison.encode!(value),
+        headers: [{:"Content-Type", "application/json"}]
+      )
+    )
+  end
+
+  def add_param(request, :file, name, path) do
+    request
+    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
+  end
+
+  def add_param(request, :form, name, value) do
+    request
+    |> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
+  end
+
+  def add_param(request, location, key, value) do
+    Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
+  end
+end

lib/pleroma/plugs/oauth_plug.ex

 defmodule Pleroma.Plugs.OAuthPlug do
   import Plug.Conn
-  alias Pleroma.User
-  alias Pleroma.Repo
-  alias Pleroma.Web.OAuth.Token
+  import Ecto.Query
 
-  def init(options) do
-    options
-  end
+  alias Pleroma.{
+    User,
+    Repo,
+    Web.OAuth.Token
+  }
+
+  @realm_reg Regex.compile!("Bearer\:?\s+(.*)$", "i")
+
+  def init(options), do: options
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
   def call(conn, _) do
-    token =
-      case get_req_header(conn, "authorization") do
-        ["Bearer " <> header] -> header
-        _ -> get_session(conn, :oauth_token)
-      end
-
-    with token when not is_nil(token) <- token,
-         %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
-         %User{} = user <- Repo.get(User, user_id),
-         false <- !!user.info.deactivated do
-      conn
-      |> assign(:user, user)
+    with {:ok, token} <- fetch_token(conn),
+         {:ok, user} <- fetch_user(token) do
+      assign(conn, :user, user)
     else
       _ -> conn
     end
   end
+
+  # Gets user by token
+  #
+  @spec fetch_user(String.t()) :: {:ok, User.t()} | nil
+  defp fetch_user(token) do
+    query = from(q in Token, where: q.token == ^token, preload: [:user])
+
+    with %Token{user: %{info: %{deactivated: false} = _} = user} <- Repo.one(query) do
+      {:ok, user}
+    end
+  end
+
+  # Gets token from session by :oauth_token key
+  #
+  @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()}
+  defp fetch_token_from_session(conn) do
+    case get_session(conn, :oauth_token) do
+      nil -> :no_token_found
+      token -> {:ok, token}
+    end
+  end
+
+  # Gets token from headers
+  #
+  @spec fetch_token(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()}
+  defp fetch_token(%Plug.Conn{} = conn) do
+    headers = get_req_header(conn, "authorization")
+
+    with :no_token_found <- fetch_token(headers),
+         do: fetch_token_from_session(conn)
+  end
+
+  @spec fetch_token(Keyword.t()) :: :no_token_found | {:ok, String.t()}
+  defp fetch_token([]), do: :no_token_found
+
+  defp fetch_token([token | tail]) do
+    trimmed_token = String.trim(token)
+
+    case Regex.run(@realm_reg, trimmed_token) do
+      [_, match] -> {:ok, String.trim(match)}
+      _ -> fetch_token(tail)
+    end
+  end
 end

lib/pleroma/uploaders/mdii.ex

@@ -20,7 +20,7 @@ def put_file(upload) do
     extension = String.split(upload.name, ".") |> List.last()
     query = "#{cgi}?#{extension}"
 
-    with {:ok, %{status_code: 200, body: body}} <- @httpoison.post(query, file_data) do
+    with {:ok, %{status: 200, body: body}} <- @httpoison.post(query, file_data) do
       remote_file_name = String.split(body) |> List.first()
       public_url = "#{files}/#{remote_file_name}.#{extension}"
       {:ok, {:url, public_url}}

lib/pleroma/uploaders/swift/keystone.ex

@@ -25,10 +25,10 @@ def get_token() do
            ["Content-Type": "application/json"],
            hackney: [:insecure]
          ) do
-      {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
+      {:ok, %Tesla.Env{status: 200, body: body}} ->
         body["access"]["token"]["id"]
 
-      {:ok, %HTTPoison.Response{status_code: _}} ->
+      {:ok, %Tesla.Env{status: _}} ->
         ""
     end
   end

lib/pleroma/uploaders/swift/swift.ex

@@ -13,10 +13,10 @@ def upload_file(filename, body, content_type) do
     token = Pleroma.Uploaders.Swift.Keystone.get_token()
 
     case put("#{filename}", body, "X-Auth-Token": token, "Content-Type": content_type) do
-      {:ok, %HTTPoison.Response{status_code: 201}} ->
+      {:ok, %Tesla.Env{status: 201}} ->
         {:ok, {:file, filename}}
 
-      {:ok, %HTTPoison.Response{status_code: 401}} ->
+      {:ok, %Tesla.Env{status: 401}} ->
         {:error, "Unauthorized, Bad Token"}
 
       {:error, _} ->

lib/pleroma/user.ex

@@ -4,6 +4,8 @@ defmodule Pleroma.User do
   import Ecto.{Changeset, Query}
   alias Pleroma.{Repo, User, Object, Web, Activity, Notification}
   alias Comeonin.Pbkdf2
+  alias Pleroma.Formatter
+  alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
   alias Pleroma.Web.{OStatus, Websub, OAuth}
   alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
 
@@ -175,6 +177,7 @@ def register_changeset(struct, params \\ %{}) do
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
       |> validate_length(:name, min: 1, max: 100)
+      |> put_change(:info, %Pleroma.User.Info{})
 
     if changeset.valid? do
       hashed = Pbkdf2.hashpwsalt(changeset.changes[:password])
@@ -802,4 +805,18 @@ def wait_and_refresh(timeout, %User{} = a, %User{} = b) do
         :error
     end
   end
+
+  def parse_bio(bio, user \\ %User{info: %{source_data: %{}}}) do
+    mentions = Formatter.parse_mentions(bio)
+    tags = Formatter.parse_tags(bio)
+
+    emoji =
+      (user.info.source_data["tag"] || [])
+      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
+        {String.trim(name, ":"), url}
+      end)
+
+    CommonUtils.format_input(bio, mentions, tags, "text/plain") |> Formatter.emojify(emoji)
+  end
 end

lib/pleroma/user/info.ex

@@ -4,7 +4,7 @@ defmodule Pleroma.User.Info do
 
   embedded_schema do
     field(:banner, :map, default: %{})
-    field(:background, :string, default: nil)
+    field(:background, :map, default: %{})
     field(:source_data, :map, default: %{})
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
@@ -137,6 +137,7 @@ def profile_update(info, params) do
       :default_scope,
       :banner,
       :hide_network
+      :background
     ])
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -762,7 +762,7 @@ def fetch_and_contain_remote_object_from_id(id) do
     Logger.info("Fetching #{id} via AP")
 
     with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
+         {:ok, %{body: body, status: code}} when code in 200..299 <-
            @httpoison.get(
              id,
              [Accept: "application/activity+json"],

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -23,7 +23,7 @@ defp check_reject(%{host: actor_host} = _actor_info, object) do
 
   defp check_media_removal(
          %{host: actor_host} = _actor_info,
-         %{"type" => "Create", "object" => %{"attachement" => child_attachment}} = object
+         %{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object
        )
        when length(child_attachment) > 0 do
     object =

lib/pleroma/web/federator/retry_queue.ex

@@ -17,7 +17,15 @@ def init(args) do
   end
 
   def start_link() do
-    GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+    enabled = Pleroma.Config.get([:retry_queue, :enabled], false)
+
+    if enabled do
+      Logger.info("Starting retry queue")
+      GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+    else
+      Logger.info("Retry queue disabled")
+      :ignore
+    end
   end
 
   def enqueue(data, transport, retries \\ 0) do

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -55,7 +55,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
     user_params =
       %{}
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio)
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end)
       |> add_if_present(params, "avatar", :avatar, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
@@ -1181,7 +1181,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
       user = user.nickname
       url = String.replace(api, "{{host}}", host) |> String.replace("{{user}}", user)
 
-      with {:ok, %{status_code: 200, body: body}} <-
+      with {:ok, %{status: 200, body: body}} <-
              @httpoison.get(url, [], timeout: timeout, recv_timeout: timeout),
            {:ok, data} <- Jason.decode(body) do
         data2 =

lib/pleroma/web/media_proxy/controller.ex

@@ -5,12 +5,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   @default_proxy_opts [max_body_length: 25 * 1_048_576]
 
   def remote(conn, params = %{"sig" => sig64, "url" => url64}) do