Merge branch 'develop' into feature/compat/push-subscriptions

# Conflicts:
#	lib/pleroma/application.ex
#	lib/pleroma/plugs/oauth_plug.ex

.gitignore

@@ -6,6 +6,9 @@
 /uploads
 /test/uploads
 /.elixir_ls
+/test/fixtures/test_tmp.txt
+/test/fixtures/image_tmp.jpg
+/doc
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*
@@ -28,4 +31,4 @@ erl_crash.dump
 .env
 
 # Editor config
-/.vscode
\ No newline at end of file
+/.vscode

config/config.exs

@@ -10,18 +10,18 @@
 
 config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
 
+# Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
-  strip_exif: false
+  filters: [],
+  proxy_remote: false,
+  proxy_opts: []
 
-config :pleroma, Pleroma.Uploaders.Local,
-  uploads: "uploads",
-  uploads_url: "{{base_url}}/media/{{file}}"
+config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
 config :pleroma, Pleroma.Uploaders.S3,
   bucket: nil,
-  public_endpoint: "https://s3.amazonaws.com",
-  force_media_proxy: false
+  public_endpoint: "https://s3.amazonaws.com"
 
 config :pleroma, Pleroma.Uploaders.MDII,
   cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
@@ -72,6 +72,7 @@
 config :pleroma, :websub, Pleroma.Web.Websub
 config :pleroma, :ostatus, Pleroma.Web.OStatus
 config :pleroma, :httpoison, Pleroma.HTTP
+config :tesla, adapter: Tesla.Adapter.Hackney
 
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http, proxy_url: nil
@@ -150,9 +151,11 @@
 
 config :pleroma, :media_proxy,
   enabled: false,
-  redirect_on_failure: true
-
-# base_url: "https://cache.pleroma.social"
+  # base_url: "https://cache.pleroma.social",
+  proxy_opts: [
+    # inline_content_types: [] | false | true,
+    # http: [:insecure]
+  ]
 
 config :pleroma, :chat, enabled: true
 

config/config.md

@@ -5,11 +5,19 @@ If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherw
 
 ## Pleroma.Upload
 * `uploader`: Select which `Pleroma.Uploaders` to use
-* `strip_exif`: boolean, uses ImageMagick(!) to strip exif.
+* `filters`: List of `Pleroma.Upload.Filter` to use.
+* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
+* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
+* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
+
+Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Uploaders.Local
 * `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory
-* `uploads_url`: The URL to access a user-uploaded file, ``{{base_url}}`` is replaced to the instance URL and ``{{file}}`` to the filename. Useful when you want to proxy the media files via another host.
+
+## Pleroma.Upload.Filter.Mogrify
+
+* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", {"impode", "1"}]`.
 
 ## :uri_schemes
 * `valid_schemes`: List of the scheme part that is considered valid to be an URL
@@ -68,7 +76,8 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 
 ## :media_proxy
 * `enabled`: Enables proxying of remote media to the instance’s proxy
-* `redirect_on_failure`: Use the original URL when Media Proxy fails to get it
+* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
+* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
 
 ## :gopher
 * `enabled`: Enables the gopher interface

config/dev.exs

@@ -49,11 +49,10 @@
   hostname: "localhost",
   pool_size: 10
 
-try do
+if File.exists?("./config/dev.secret.exs") do
   import_config "dev.secret.exs"
-rescue
-  _ ->
-    IO.puts(
-      "!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
-    )
+else
+  IO.puts(
+    "!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
+  )
 end

config/test.exs

@@ -9,7 +9,7 @@
 # Print only warnings and errors during test
 config :logger, level: :warn
 
-config :pleroma, Pleroma.Upload, uploads: "test/uploads"
+config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
 
 # Configure your database
 config :pleroma, Pleroma.Repo,
@@ -25,7 +25,7 @@
 
 config :pleroma, :websub, Pleroma.Web.WebsubMock
 config :pleroma, :ostatus, Pleroma.Web.OStatusMock
-config :pleroma, :httpoison, HTTPoisonMock
+config :tesla, adapter: Tesla.Mock
 
 try do
   import_config "test.secret.exs"

installation/pleroma.nginx

@@ -70,10 +70,12 @@ server {
         client_max_body_size 16m;
     }
 
-    location /proxy {
+    location ~ ^/(media|proxy) {
         proxy_cache pleroma_media_cache;
         proxy_cache_lock on;
         proxy_ignore_client_abort on;
+        proxy_buffering off;
+        chunked_transfer_encoding on;
         proxy_pass http://localhost:4000;
     }
 }

lib/mix/tasks/make_moderator.ex

@@ -8,7 +8,7 @@ defmodule Mix.Tasks.SetModerator do
   """
 
   use Mix.Task
-  import Mix.Ecto
+  import Ecto.Changeset
   alias Pleroma.{Repo, User}
 
   def run([nickname | rest]) do
@@ -21,14 +21,15 @@ def run([nickname | rest]) do
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("is_moderator", !!moderator)
+      info_cng = User.Info.admin_api_update(user.info, %{is_moderator: !!moderator})
 
-      cng = User.info_changeset(user, %{info: info})
-      {:ok, user} = User.update_and_set_cache(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("Moderator status of #{nickname}: #{user.info["is_moderator"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Moderator status of #{nickname}: #{user.info.is_moderator}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/mix/tasks/migrate_local_uploads.ex

+defmodule Mix.Tasks.MigrateLocalUploads do
+  use Mix.Task
+  import Mix.Ecto
+  alias Pleroma.{Upload, Uploaders.Local, Uploaders.S3}
+  require Logger
+
+  @log_every 50
+  @shortdoc "Migrate uploads from local to remote storage"
+
+  def run([target_uploader | args]) do
+    delete? = Enum.member?(args, "--delete")
+    Application.ensure_all_started(:pleroma)
+
+    local_path = Pleroma.Config.get!([Local, :uploads])
+    uploader = Module.concat(Pleroma.Uploaders, target_uploader)
+
+    unless Code.ensure_loaded?(uploader) do
+      raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
+    end
+
+    target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
+
+    unless target_enabled? do
+      Pleroma.Config.put([Upload, :uploader], uploader)
+    end
+
+    Logger.info("Migrating files from local #{local_path} to #{to_string(uploader)}")
+
+    if delete? do
+      Logger.warn(
+        "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
+      )
+
+      :timer.sleep(:timer.seconds(5))
+    end
+
+    uploads =
+      File.ls!(local_path)
+      |> Enum.map(fn id ->
+        root_path = Path.join(local_path, id)
+
+        cond do
+          File.dir?(root_path) ->
+            files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
+
+            case List.first(files) do
+              {id, file, path} ->
+                {%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
+                 root_path}
+
+              _ ->
+                nil
+            end
+
+          File.exists?(root_path) ->
+            file = Path.basename(id)
+            [hash, ext] = String.split(id, ".")
+            {%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
+
+          true ->
+            nil
+        end
+      end)
+      |> Enum.filter(& &1)
+
+    total_count = length(uploads)
+    Logger.info("Found #{total_count} uploads")
+
+    uploads
+    |> Task.async_stream(
+      fn {upload, root_path} ->
+        case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
+          {:ok, _} ->
+            if delete?, do: File.rm_rf!(root_path)
+            Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
+            :ok
+
+          error ->
+            Logger.error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
+        end
+      end,
+      timeout: 150_000
+    )
+    |> Stream.chunk_every(@log_every)
+    |> Enum.reduce(0, fn done, count ->
+      count = count + length(done)
+      Logger.info("Uploaded #{count}/#{total_count} files")
+      count
+    end)
+
+    Logger.info("Done!")
+  end
+
+  def run(_) do
+    Logger.error("Usage: migrate_local_uploads S3|Swift [--delete]")
+  end
+end

lib/mix/tasks/sample_psql.eex

@@ -4,3 +4,4 @@ CREATE DATABASE pleroma_dev OWNER pleroma;
 --Extensions made by ecto.migrate that need superuser access
 CREATE EXTENSION IF NOT EXISTS citext;
 CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

lib/mix/tasks/set_admin.ex

 defmodule Mix.Tasks.SetAdmin do
   use Mix.Task
+  import Ecto.Changeset
   alias Pleroma.User
 
   @doc """
@@ -9,21 +10,22 @@ defmodule Mix.Tasks.SetAdmin do
   def run([nickname | rest]) do
     Application.ensure_all_started(:pleroma)
 
-    status =
+    admin =
       case rest do
-        [status] -> status == "true"
+        [admin] -> admin == "true"
         _ -> true
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("is_admin", !!status)
+      info_cng = User.Info.admin_api_update(user.info, %{is_admin: !!admin})
 
-      cng = User.info_changeset(user, %{info: info})
-      {:ok, user} = User.update_and_set_cache(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("Admin status of #{nickname}: #{user.info["is_admin"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Admin status of #{nickname}: #{user.info.is_admin}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/mix/tasks/set_locked.ex

@@ -10,11 +10,11 @@ defmodule Mix.Tasks.SetLocked do
   """
 
   use Mix.Task
-  import Mix.Ecto
+  import Ecto.Changeset
   alias Pleroma.{Repo, User}
 
   def run([nickname | rest]) do
-    ensure_started(Repo, [])
+    Application.ensure_all_started(:pleroma)
 
     locked =
       case rest do
@@ -23,14 +23,15 @@ def run([nickname | rest]) do
       end
 
     with %User{local: true} = user <- User.get_by_nickname(nickname) do
-      info =
-        user.info
-        |> Map.put("locked", !!locked)
+      info_cng = User.Info.profile_update(user.info, %{locked: !!locked})
 
-      cng = User.info_changeset(user, %{info: info})
-      user = Repo.update!(cng)
+      user_cng =
+        Ecto.Changeset.change(user)
+        |> put_embed(:info, info_cng)
 
-      IO.puts("locked status of #{nickname}: #{user.info["locked"]}")
+      {:ok, user} = User.update_and_set_cache(user_cng)
+
+      IO.puts("Locked status of #{nickname}: #{user.info.locked}")
     else
       _ ->
         IO.puts("No local user #{nickname}")

lib/pleroma/application.ex

 defmodule Pleroma.Application do
   use Application
+  import Supervisor.Spec
 
   @name "Pleroma"
   @version Mix.Project.config()[:version]
@@ -7,11 +8,15 @@ def name, do: @name
   def version, do: @version
   def named_version(), do: @name <> " " <> @version
 
+  def user_agent() do
+    info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+    named_version() <> "; " <> info
+  end
+
   # See http://elixir-lang.org/docs/stable/elixir/Application.html
   # for more information on OTP Applications
   @env Mix.env()
   def start(_type, _args) do
-    import Supervisor.Spec
     import Cachex.Spec
 
     # Define workers and child supervisors to be supervised
@@ -20,10 +25,6 @@ def start(_type, _args) do
         # Start the Ecto repository
         supervisor(Pleroma.Repo, []),
         worker(Pleroma.Emoji, []),
-        # Start the endpoint when the application starts
-        supervisor(Pleroma.Web.Endpoint, []),
-        # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
-        # worker(Pleroma.Worker, [arg1, arg2, arg3]),
         worker(
           Cachex,
           [
@@ -63,21 +64,18 @@ def start(_type, _args) do
           ],
           id: :cachex_idem
         ),
-        worker(Pleroma.Web.Federator, []),
         worker(Pleroma.Web.Federator.RetryQueue, []),
-        worker(Pleroma.Gopher.Server, []),
+        worker(Pleroma.Web.Federator, []),
         worker(Pleroma.Stats, []),
         worker(Pleroma.Web.Push, [])
       ] ++
-        if @env == :test,
-          do: [],
-          else:
-            [worker(Pleroma.Web.Streamer, [])] ++
-              if(
-                !chat_enabled(),
-                do: [],
-                else: [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
-              )
+        streamer_child() ++
+        chat_child() ++
+        [
+          # Start the endpoint when the application starts
+          supervisor(Pleroma.Web.Endpoint, []),
+          worker(Pleroma.Gopher.Server, [])
+        ]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
@@ -85,7 +83,20 @@ def start(_type, _args) do
     Supervisor.start_link(children, opts)
   end
 
-  defp chat_enabled do
-    Application.get_env(:pleroma, :chat, []) |> Keyword.get(:enabled)
+  if Mix.env() == :test do
+    defp streamer_child(), do: []
+    defp chat_child(), do: []
+  else
+    defp streamer_child() do
+      [worker(Pleroma.Web.Streamer, [])]
+    end
+
+    defp chat_child() do
+      if Pleroma.Config.get([:chat, :enabled]) do
+        [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
+      else
+        []
+      end
+    end
   end
 end

lib/pleroma/config.ex

@@ -39,4 +39,18 @@ def put([parent_key | keys], value) do
   def put(key, value) do
     Application.put_env(:pleroma, key, value)
   end
+
+  def delete([key]), do: delete(key)
+
+  def delete([parent_key | keys]) do
+    {_, parent} =
+      Application.get_env(:pleroma, parent_key)
+      |> get_and_update_in(keys, fn _ -> :pop end)
+
+    Application.put_env(:pleroma, parent_key, parent)
+  end
+
+  def delete(key) do
+    Application.delete_env(:pleroma, key)
+  end
 end

lib/pleroma/formatter.ex

@@ -114,10 +114,10 @@ def add_user_links({subs, text}, mentions) do
 
     subs =
       subs ++
-        Enum.map(mentions, fn {match, %User{ap_id: ap_id, info: info}, uuid} ->
+        Enum.map(mentions, fn {match, %User{id: id, ap_id: ap_id, info: info}, uuid} ->
           ap_id =
-            if is_binary(info["source_data"]["url"]) do
-              info["source_data"]["url"]
+            if is_binary(info.source_data["url"]) do
+              info.source_data["url"]
             else
               ap_id
             end
@@ -125,7 +125,7 @@ def add_user_links({subs, text}, mentions) do
           short_match = String.split(match, "@") |> tl() |> hd()
 
           {uuid,
-           "<span><a class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
+           "<span><a data-user='#{id}' class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
         end)
 
     {subs, uuid_text}
@@ -147,7 +147,11 @@ def add_hashtag_links({subs, text}, tags) do
     subs =
       subs ++
         Enum.map(tags, fn {tag_text, tag, uuid} ->
-          url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{tag_text}</a>"
+          url =
+            "<a data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
+              tag_text
+            }</a>"
+
           {uuid, url}
         end)
 

lib/pleroma/gopher/server.ex

@@ -6,28 +6,29 @@ def start_link() do
     config = Pleroma.Config.get(:gopher, [])
     ip = Keyword.get(config, :ip, {0, 0, 0, 0})
     port = Keyword.get(config, :port, 1234)
-    GenServer.start_link(__MODULE__, [ip, port], [])
-  end
 
-  def init([ip, port]) do
-    if Pleroma.Config.get([:gopher, :enabled], false) do
-      Logger.info("Starting gopher server on #{port}")
-
-      :ranch.start_listener(
-        :gopher,
-        100,
-        :ranch_tcp,
-        [port: port],
-        __MODULE__.ProtocolHandler,
-        []
-      )
-
-      {:ok, %{ip: ip, port: port}}
+    if Keyword.get(config, :enabled, false) do
+      GenServer.start_link(__MODULE__, [ip, port], [])
     else
       Logger.info("Gopher server disabled")
-      {:ok, nil}
+      :ignore
     end
   end
+
+  def init([ip, port]) do
+    Logger.info("Starting gopher server on #{port}")
+
+    :ranch.start_listener(
+      :gopher,
+      100,
+      :ranch_tcp,
+      [port: port],
+      __MODULE__.ProtocolHandler,
+      []
+    )
+
+    {:ok, %{ip: ip, port: port}}
+  end
 end
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do

lib/pleroma/html.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   Meta.strip_comments()
 
   # links
-  Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes)
+  Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title"])
 
   # paragraphs and linebreaks
@@ -86,7 +86,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.remove_cdata_sections_before_scrub()
   Meta.strip_comments()
 
-  Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes)
+  Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title"])
 
   Meta.allow_tag_with_these_attributes("abbr", ["title"])

lib/pleroma/http/connection.ex

+defmodule Pleroma.HTTP.Connection do
+  @moduledoc """
+  Connection for http-requests.
+  """
+
+  @hackney_options [pool: :default]
+  @adapter Application.get_env(:tesla, :adapter)
+
+  @doc """
+  Configure a client connection
+
+  # Returns
+
+  Tesla.Env.client
+  """
+  @spec new(Keyword.t()) :: Tesla.Env.client()
+  def new(opts \\ []) do
+    Tesla.client([], {@adapter, hackney_options(opts)})
+  end
+
+  # fetch Hackney options
+  #
+  defp hackney_options(opts \\ []) do
+    options = Keyword.get(opts, :adapter, [])
+    @hackney_options ++ options
+  end
+end

lib/pleroma/http/http.ex

 defmodule Pleroma.HTTP do
-  require HTTPoison
+  @moduledoc """
 
+  """
+
+  alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.RequestBuilder, as: Builder
+
+  @doc """
+  Builds and perform http request.
+
+  # Arguments:
+  `method` - :get, :post, :put, :delete
+  `url`
+  `body`
+  `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
+  `options` - custom, per-request middleware or adapter options
+
+  # Returns: