adding gun adapter

CHANGELOG.md

@@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for custom Elixir modules (such as MRF policies)
 - User settings: Add _This account is a_ option.
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
+- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires OTP version older that 22.2, otherwise pleroma won’t start. For hackney OTP update is not required.
 <details>
   <summary>API Changes</summary>
 

config/config.exs

@@ -58,20 +58,6 @@
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
 
-config :pleroma, :hackney_pools,
-  federation: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  media: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  upload: [
-    max_connections: 25,
-    timeout: 300_000
-  ]
-
 # Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
@@ -185,20 +171,12 @@
 }
 
 config :tesla, adapter: Tesla.Adapter.Hackney
-
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
   send_user_agent: true,
   user_agent: :default,
-  adapter: [
-    ssl_options: [
-      # Workaround for remote server certificate chain issues
-      partial_chain: &:hackney_connect.partial_chain/1,
-      # We don't support TLS v1.3 yet
-      versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
-    ]
-  ]
+  adapter: []
 
 config :pleroma, :instance,
   name: "Pleroma",
@@ -612,6 +590,49 @@
 
 config :pleroma, configurable_from_database: false
 
+config :pleroma, :connections_pool,
+  receive_connection_timeout: 250,
+  max_connections: 250,
+  retry: 5,
+  retry_timeout: 100,
+  await_up_timeout: 5_000
+
+config :pleroma, :pools,
+  federation: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  media: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  upload: [
+    size: 25,
+    max_overflow: 5,
+    timeout: 300_000
+  ],
+  default: [
+    size: 10,
+    max_overflow: 2,
+    timeout: 10_000
+  ]
+
+config :pleroma, :hackney_pools,
+  federation: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  upload: [
+    max_connections: 25,
+    timeout: 300_000
+  ]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -2728,7 +2728,7 @@
         key: :adapter,
         type: :module,
         description: "Tesla adapter",
-        suggestions: [Tesla.Adapter.Hackney]
+        suggestions: [Tesla.Adapter.Hackney, Tesla.Adapter.Gun]
       }
     ]
   },

config/test.exs

@@ -94,6 +94,8 @@
 
 config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
 
+config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/API/admin_api.md

@@ -731,6 +731,8 @@ Some modifications are necessary to save the config settings correctly:
 Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
 - all settings inside these keys:
   - `:hackney_pools`
+  - `:connections_pool`
+  - `:pools`
   - `:chat`
 - partially settings inside these keys:
   - `:seconds_valid` in `Pleroma.Captcha`

docs/configuration/cheatsheet.md

@@ -368,8 +368,7 @@ Available caches:
 * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
 * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
 * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
-* `adapter`: array of hackney options
-
+* `adapter`: array of adapter options
 
 ### :hackney_pools
 
@@ -388,6 +387,39 @@ For each pool, the options are:
 * `timeout` - retention duration for connections
 
 
+### :connections_pool
+
+*For `gun` adapter*
+
+Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
+
+* `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms.
+* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
+* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5.
+* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 100ms.
+* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
+
+### :pools
+
+*For `gun` adapter*
+
+Advanced settings for workers pools.
+
+There's four pools used:
+
+* `:federation` for the federation jobs.
+  You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
+* `:media` for rich media, media proxy
+* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
+* `:default` for other requests
+
+For each pool, the options are:
+
+* `:size` - how much workers the pool can hold
+* `:timeout` - timeout while `gun` will wait for response
+* `:max_overflow` - additional workers if pool is under load
+
+
 ## Captcha
 
 ### Pleroma.Captcha

lib/mix/tasks/pleroma/benchmark.ex

@@ -74,4 +74,43 @@ def run(["render_timeline", nickname | _] = args) do
       inputs: inputs
     )
   end
+
+  def run(["adapters"]) do
+    start_pleroma()
+
+    :ok =
+      Pleroma.Pool.Connections.open_conn(
+        "https://httpbin.org/stream-bytes/1500",
+        :gun_connections
+      )
+
+    Process.sleep(1_500)
+
+    Benchee.run(
+      %{
+        "Without conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool, receive_conn: false]
+            )
+        end,
+        "Without conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [receive_conn: false]
+            )
+        end,
+        "With reused conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool]
+            )
+        end,
+        "With reused conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+        end
+      },
+      parallel: 10
+    )
+  end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -4,13 +4,13 @@
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages emoji packs"
   @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, [], []} = parse_global_opts(args)
 
@@ -36,8 +36,7 @@ def run(["ls-packs" | args]) do
   end
 
   def run(["get-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, pack_names, []} = parse_global_opts(args)
 
@@ -135,7 +134,7 @@ def run(["get-packs" | args]) do
   end
 
   def run(["gen-pack", src]) do
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     proposed_name = Path.basename(src) |> Path.rootname()
     name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))

lib/pleroma/application.ex

@@ -3,8 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
-  import Cachex.Spec
   use Application
+
+  import Cachex.Spec
+
+  alias Pleroma.Config
+
   require Logger
 
   @name Mix.Project.config()[:name]
@@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
   def user_agent do
-    case Pleroma.Config.get([:http, :user_agent], :default) do
+    case Config.get([:http, :user_agent], :default) do
       :default ->
-        info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
         named_version() <> "; " <> info
 
       custom ->
@@ -32,7 +36,7 @@ def user_agent do
   # for more information on OTP Applications
   def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.DeprecationWarnings.warn()
+    Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.Repo.check_migrations_applied!()
     setup_instrumenters()
@@ -42,17 +46,17 @@ def start(_type, _args) do
     children =
       [
         Pleroma.Repo,
-        Pleroma.Config.TransferTask,
+        Config.TransferTask,
         Pleroma.Emoji,
         Pleroma.Captcha,
         Pleroma.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
-        hackney_pool_children() ++
+        http_pools_children(Config.get(:env)) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
-          {Oban, Pleroma.Config.get(Oban)}
+          {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         streamer_child(@env) ++
@@ -62,6 +66,18 @@ def start(_type, _args) do
           Pleroma.Gopher.Server
         ]
 
+    case Pleroma.OTPVersion.check_version() do
+      :ok -> :ok
+      {:error, version} -> raise "
+        !!!OTP VERSION WARNING!!!
+        You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
+        "
+      :undefined -> raise "
+        !!!OTP VERSION WARNING!!!
+        To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+        "
+    end
+
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
@@ -69,7 +85,7 @@ def start(_type, _args) do
   end
 
   def load_custom_modules do
-    dir = Pleroma.Config.get([:modules, :runtime_dir])
+    dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
       dir
@@ -110,20 +126,6 @@ defp setup_instrumenters do
     Pleroma.Web.Endpoint.Instrumenter.setup()
   end
 
-  def enabled_hackney_pools do
-    [:media] ++
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        [:federation]
-      else
-        []
-      end ++
-      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload]
-      else
-        []
-      end
-  end
-
   defp cachex_children do
     [
       build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@@ -145,7 +147,7 @@ defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
   defp seconds_valid_interval,
-    do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
   defp build_cachex(type, opts),
     do: %{
@@ -154,7 +156,7 @@ defp build_cachex(type, opts),
       type: :worker
     }
 
-  defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
+  defp chat_enabled?, do: Config.get([:chat, :enabled])
 
   defp streamer_child(:test), do: []
 
@@ -168,13 +170,6 @@ defp chat_child(_env, true) do
 
   defp chat_child(_, _), do: []
 
-  defp hackney_pool_children do
-    for pool <- enabled_hackney_pools() do
-      options = Pleroma.Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
   defp task_children(:test) do
     [
       %{
@@ -199,4 +194,37 @@ defp task_children(_) do
       }
     ]
   end
+
+  # start hackney and gun pools in tests
+  defp http_pools_children(:test) do
+    hackney_options = Config.get([:hackney_pools, :federation])
+    hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
+    [hackney_pool, Pleroma.Pool.Supervisor]
+  end
+
+  defp http_pools_children(_) do
+    :tesla
+    |> Application.get_env(:adapter)
+    |> http_pools()
+  end
+
+  defp http_pools(Tesla.Adapter.Hackney) do
+    pools = [:federation, :media]
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    for pool <- pools do
+      options = Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
+
+  defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor]
+
+  defp http_pools(_), do: []
 end

lib/pleroma/config/config_db.ex

@@ -278,8 +278,6 @@ defp do_convert({:proxy_url, {type, host, port}}) do
     }
   end
 
-  defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
-
   defp do_convert(entity) when is_tuple(entity) do
     value =
       entity
@@ -323,15 +321,6 @@ defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}
     {:proxy_url, {do_transform_string(type), parse_host(host), port}}
   end
 
-  defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
-    {partial_chain, []} =
-      entity
-      |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
-      |> Code.eval_string()
-
-    {:partial_chain, partial_chain}
-  end
-
   defp do_transform(%{"tuple" => entity}) do
     Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
   end

lib/pleroma/config/transfer_task.ex

@@ -18,7 +18,10 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, Oban},
     {:pleroma, :rate_limit},
     {:pleroma, :markup},
-    {:plerome, :streamer}
+    {:pleroma, :streamer},
+    {:pleroma, :pools},
+    {:pleroma, :connections_pool},
+    {:tesla, :adapter}
   ]
 
   @reboot_time_subkeys [
@@ -74,6 +77,28 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
     end
   end
 
+  defp group_for_restart(:logger, key, _, merged_value) do
+    # change logger configuration in runtime, without restart
+    if Keyword.keyword?(merged_value) and
+         key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
+      Logger.configure_backend(key, merged_value)
+    else
+      Logger.configure([{key, merged_value}])
+    end
+
+    nil
+  end
+
+  defp group_for_restart(:tesla, _, _, _), do: :pleroma
+
+  defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
+
+  defp group_for_restart(group, key, value, _) do
+    if pleroma_need_restart?(group, key, value) do
+      group
+    end
+  end
+
   defp merge_and_update(setting) do
     try do
       key = ConfigDB.from_string(setting.key)
@@ -95,21 +120,7 @@ defp merge_and_update(setting) do
 
       :ok = update_env(group, key, merged_value)
 
-      if group != :logger do
-        if group != :pleroma or pleroma_need_restart?(group, key, value) do
-          group
-        end
-      else
-        # change logger configuration in runtime, without restart
-        if Keyword.keyword?(merged_value) and
-             key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
-          Logger.configure_backend(key, merged_value)
-        else
-          Logger.configure([{key, merged_value}])
-        end
-
-        nil
-      end
+      group_for_restart(group, key, value, merged_value)
     rescue
       error ->
         error_msg =

lib/pleroma/gun/api.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.API do
+  @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
+  @callback info(pid()) :: map()
+  @callback close(pid()) :: :ok
+  @callback await_up(pid) :: {:ok, atom()} | {:error, atom()}
+  @callback connect(pid(), map()) :: reference()
+  @callback await(pid(), reference()) :: {:response, :fin, 200, []}
+
+  def open(host, port, opts), do: api().open(host, port, opts)
+
+  def info(pid), do: api().info(pid)
+
+  def close(pid), do: api().close(pid)
+
+  def await_up(pid), do: api().await_up(pid)
+
+  def connect(pid, opts), do: api().connect(pid, opts)
+
+  def await(pid, ref), do: api().await(pid, ref)
+
+  defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun)
+end

lib/pleroma/gun/api/mock.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.API.Mock do
+  @behaviour Pleroma.Gun.API
+
+  alias Pleroma.Gun.API
+
+  @impl API
+  def open('some-domain.com', 443, _) do
+    {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
+
+    Registry.register(API.Mock, conn_pid, %{
+      origin_scheme: "https",
+      origin_host: 'some-domain.com',
+      origin_port: 443
+    })
+
+    {:ok, conn_pid}
+  end
+
+  @impl API
+  def open(ip, port, _)
+      when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and
+             port in [80, 443] do
+    {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
+
+    scheme = if port == 443, do: "https", else: "http"
+
+    Registry.register(API.Mock, conn_pid, %{
+      origin_scheme: scheme,
+      origin_host: ip,
+      origin_port: port
+    })
+
+    {:ok, conn_pid}
+  end
+
+  @impl API
+  def open('localhost', 1234, %{
+        protocols: [:socks],
+        proxy: {:socks5, 'localhost', 1234},
+        socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5}
+      }) do
+    {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
+
+    Registry.register(API.Mock, conn_pid, %{
+      origin_scheme: "http",
+      origin_host: 'proxy-socks.com',
+      origin_port: 80
+    })
+
+    {:ok, conn_pid}
+  end
+
+  @impl API
+  def open('localhost', 1234, %{
+        protocols: [:socks],
+        proxy: {:socks4, 'localhost', 1234},
+        socks_opts: %{
+          host: 'proxy-socks.com',
+          port: 443,
+          protocols: [:http2],
+          tls_opts: [],
+          transport: :tls,
+          version: 4
+        }
+      }) do
+    {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)