Verified Commit 514c8992 authored by Alexander Strizhakov's avatar Alexander Strizhakov
Browse files

adding gun adapter

parent 962eb8d4
......@@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Support for custom Elixir modules (such as MRF policies)
- User settings: Add _This account is a_ option.
- OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires OTP version older that 22.2, otherwise pleroma won’t start. For hackney OTP update is not required.
<details>
<summary>API Changes</summary>
......
......@@ -58,20 +58,6 @@
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
......@@ -185,20 +171,12 @@
}
config :tesla, adapter: Tesla.Adapter.Hackney
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
proxy_url: nil,
send_user_agent: true,
user_agent: :default,
adapter: [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
]
]
adapter: []
config :pleroma, :instance,
name: "Pleroma",
......@@ -612,6 +590,49 @@
config :pleroma, configurable_from_database: false
config :pleroma, :connections_pool,
receive_connection_timeout: 250,
max_connections: 250,
retry: 5,
retry_timeout: 100,
await_up_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
media: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
upload: [
size: 25,
max_overflow: 5,
timeout: 300_000
],
default: [
size: 10,
max_overflow: 2,
timeout: 10_000
]
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
......@@ -2728,7 +2728,7 @@
key: :adapter,
type: :module,
description: "Tesla adapter",
suggestions: [Tesla.Adapter.Hackney]
suggestions: [Tesla.Adapter.Hackney, Tesla.Adapter.Gun]
}
]
},
......
......@@ -94,6 +94,8 @@
config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock
if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs"
else
......
......@@ -731,6 +731,8 @@ Some modifications are necessary to save the config settings correctly:
Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
- all settings inside these keys:
- `:hackney_pools`
- `:connections_pool`
- `:pools`
- `:chat`
- partially settings inside these keys:
- `:seconds_valid` in `Pleroma.Captcha`
......
......@@ -368,8 +368,7 @@ Available caches:
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
* `adapter`: array of hackney options
* `adapter`: array of adapter options
### :hackney_pools
......@@ -388,6 +387,39 @@ For each pool, the options are:
* `timeout` - retention duration for connections
### :connections_pool
*For `gun` adapter*
Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
* `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms.
* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5.
* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 100ms.
* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
### :pools
*For `gun` adapter*
Advanced settings for workers pools.
There's four pools used:
* `:federation` for the federation jobs.
You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
* `:media` for rich media, media proxy
* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
* `:default` for other requests
For each pool, the options are:
* `:size` - how much workers the pool can hold
* `:timeout` - timeout while `gun` will wait for response
* `:max_overflow` - additional workers if pool is under load
## Captcha
### Pleroma.Captcha
......
......@@ -74,4 +74,43 @@ def run(["render_timeline", nickname | _] = args) do
inputs: inputs
)
end
def run(["adapters"]) do
start_pleroma()
:ok =
Pleroma.Pool.Connections.open_conn(
"https://httpbin.org/stream-bytes/1500",
:gun_connections
)
Process.sleep(1_500)
Benchee.run(
%{
"Without conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool, receive_conn: false]
)
end,
"Without conn and with pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [receive_conn: false]
)
end,
"With reused conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool]
)
end,
"With reused conn and with pool" => fn ->
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
end
},
parallel: 10
)
end
end
......@@ -4,13 +4,13 @@
defmodule Mix.Tasks.Pleroma.Emoji do
use Mix.Task
import Mix.Pleroma
@shortdoc "Manages emoji packs"
@moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
def run(["ls-packs" | args]) do
Mix.Pleroma.start_pleroma()
Application.ensure_all_started(:hackney)
start_pleroma()
{options, [], []} = parse_global_opts(args)
......@@ -36,8 +36,7 @@ def run(["ls-packs" | args]) do
end
def run(["get-packs" | args]) do
Mix.Pleroma.start_pleroma()
Application.ensure_all_started(:hackney)
start_pleroma()
{options, pack_names, []} = parse_global_opts(args)
......@@ -135,7 +134,7 @@ def run(["get-packs" | args]) do
end
def run(["gen-pack", src]) do
Application.ensure_all_started(:hackney)
start_pleroma()
proposed_name = Path.basename(src) |> Path.rootname()
name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
......
......@@ -3,8 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Application do
import Cachex.Spec
use Application
import Cachex.Spec
alias Pleroma.Config
require Logger
@name Mix.Project.config()[:name]
......@@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version
def repository, do: @repository
def user_agent do
case Pleroma.Config.get([:http, :user_agent], :default) do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
custom ->
......@@ -32,7 +36,7 @@ def user_agent do
# for more information on OTP Applications
def start(_type, _args) do
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.DeprecationWarnings.warn()
Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.Repo.check_migrations_applied!()
setup_instrumenters()
......@@ -42,17 +46,17 @@ def start(_type, _args) do
children =
[
Pleroma.Repo,
Pleroma.Config.TransferTask,
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Captcha,
Pleroma.Plugs.RateLimiter.Supervisor
] ++
cachex_children() ++
hackney_pool_children() ++
http_pools_children(Config.get(:env)) ++
[
Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Oban, Pleroma.Config.get(Oban)}
{Oban, Config.get(Oban)}
] ++
task_children(@env) ++
streamer_child(@env) ++
......@@ -62,6 +66,18 @@ def start(_type, _args) do
Pleroma.Gopher.Server
]
case Pleroma.OTPVersion.check_version() do
:ok -> :ok
{:error, version} -> raise "
!!!OTP VERSION WARNING!!!
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
"
:undefined -> raise "
!!!OTP VERSION WARNING!!!
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
"
end
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
......@@ -69,7 +85,7 @@ def start(_type, _args) do
end
def load_custom_modules do
dir = Pleroma.Config.get([:modules, :runtime_dir])
dir = Config.get([:modules, :runtime_dir])
if dir && File.exists?(dir) do
dir
......@@ -110,20 +126,6 @@ defp setup_instrumenters do
Pleroma.Web.Endpoint.Instrumenter.setup()
end
def enabled_hackney_pools do
[:media] ++
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
[:federation]
else
[]
end ++
if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload]
else
[]
end
end
defp cachex_children do
[
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
......@@ -145,7 +147,7 @@ defp idempotency_expiration,
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp seconds_valid_interval,
do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
defp build_cachex(type, opts),
do: %{
......@@ -154,7 +156,7 @@ defp build_cachex(type, opts),
type: :worker
}
defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
defp chat_enabled?, do: Config.get([:chat, :enabled])
defp streamer_child(:test), do: []
......@@ -168,13 +170,6 @@ defp chat_child(_env, true) do
defp chat_child(_, _), do: []
defp hackney_pool_children do
for pool <- enabled_hackney_pools() do
options = Pleroma.Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp task_children(:test) do
[
%{
......@@ -199,4 +194,37 @@ defp task_children(_) do
}
]
end
# start hackney and gun pools in tests
defp http_pools_children(:test) do
hackney_options = Config.get([:hackney_pools, :federation])
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
[hackney_pool, Pleroma.Pool.Supervisor]
end
defp http_pools_children(_) do
:tesla
|> Application.get_env(:adapter)
|> http_pools()
end
defp http_pools(Tesla.Adapter.Hackney) do
pools = [:federation, :media]
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload | pools]
else
pools
end
for pool <- pools do
options = Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor]
defp http_pools(_), do: []
end
......@@ -278,8 +278,6 @@ defp do_convert({:proxy_url, {type, host, port}}) do
}
end
defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
defp do_convert(entity) when is_tuple(entity) do
value =
entity
......@@ -323,15 +321,6 @@ defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}
{:proxy_url, {do_transform_string(type), parse_host(host), port}}
end
defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
{partial_chain, []} =
entity
|> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
|> Code.eval_string()
{:partial_chain, partial_chain}
end
defp do_transform(%{"tuple" => entity}) do
Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
end
......
......@@ -18,7 +18,10 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, Oban},
{:pleroma, :rate_limit},
{:pleroma, :markup},
{:plerome, :streamer}
{:pleroma, :streamer},
{:pleroma, :pools},
{:pleroma, :connections_pool},
{:tesla, :adapter}
]
@reboot_time_subkeys [
......@@ -74,6 +77,28 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
end
end
defp group_for_restart(:logger, key, _, merged_value) do
# change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
Logger.configure_backend(key, merged_value)
else
Logger.configure([{key, merged_value}])
end
nil
end
defp group_for_restart(:tesla, _, _, _), do: :pleroma
defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
defp group_for_restart(group, key, value, _) do
if pleroma_need_restart?(group, key, value) do
group
end
end
defp merge_and_update(setting) do
try do
key = ConfigDB.from_string(setting.key)
......@@ -95,21 +120,7 @@ defp merge_and_update(setting) do
:ok = update_env(group, key, merged_value)
if group != :logger do
if group != :pleroma or pleroma_need_restart?(group, key, value) do
group
end
else
# change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
Logger.configure_backend(key, merged_value)
else
Logger.configure([{key, merged_value}])
end
nil
end
group_for_restart(group, key, value, merged_value)
rescue
error ->
error_msg =
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback await_up(pid) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
def open(host, port, opts), do: api().open(host, port, opts)
def info(pid), do: api().info(pid)
def close(pid), do: api().close(pid)
def await_up(pid), do: api().await_up(pid)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun)
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API.Mock do
@behaviour Pleroma.Gun.API
alias Pleroma.Gun.API
@impl API
def open('some-domain.com', 443, _) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "https",
origin_host: 'some-domain.com',
origin_port: 443
})
{:ok, conn_pid}
end
@impl API
def open(ip, port, _)
when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and
port in [80, 443] do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
scheme = if port == 443, do: "https", else: "http"
Registry.register(API.Mock, conn_pid, %{
origin_scheme: scheme,
origin_host: ip,
origin_port: port
})
{:ok, conn_pid}
end
@impl API
def open('localhost', 1234, %{
protocols: [:socks],
proxy: {:socks5, 'localhost', 1234},
socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5}
}) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "http",
origin_host: 'proxy-socks.com',
origin_port: 80
})
{:ok, conn_pid}
end
@impl API
def open('localhost', 1234, %{
protocols: [:socks],
proxy: {:socks4, 'localhost', 1234},
socks_opts: %{
host: 'proxy-socks.com',
port: 443,
protocols: [:http2],
tls_opts: [],
transport: :tls,
version: 4
}
}) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)