Commit 2bfdf3e0 authored by feld's avatar feld

Merge branch 'develop' into merge/develop-20200720

parents 3d960c2d 1b82b7ea
......@@ -8,9 +8,7 @@
### Environment
* Installation type:
- [ ] OTP
- [ ] From source
* Installation type (OTP or From Source):
* Pleroma version (could be found in the "Version" tab of settings in Pleroma-FE):
* Elixir version (`elixir -v` for from source installations, N/A for OTP):
* Operating system:
......
......@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- MFR policy to set global expiration for all local Create activities
- OGP rich media parser merged with TwitterCard
- Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
- Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
<details>
<summary>API Changes</summary>
......@@ -24,6 +25,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Added `pleroma.metadata.fields_limits` to /api/v1/instance
- Mastodon API: On deletion, returns the original post text.
- Mastodon API: Add `pleroma.unread_count` to the Marker entity.
- **Breaking:** Notification Settings API for suppressing notifications
has been simplified down to `block_from_strangers`.
- **Breaking:** Notification Settings API option for hiding push notification
contents has been renamed to `hide_notification_contents`
- Mastodon API: Added `pleroma.metadata.post_formats` to /api/v1/instance
</details>
<details>
......@@ -61,7 +67,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<details>
<summary>API Changes</summary>
- Mastodon API: Add pleroma.parents_visible field to statuses.
- Mastodon API: Add pleroma.parent_visible field to statuses.
- Mastodon API: Extended `/api/v1/instance`.
- Mastodon API: Support for `include_types` in `/api/v1/notifications`.
- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
......@@ -115,6 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Follow request notifications
<details>
<summary>API Changes</summary>
- Admin API: `GET /api/pleroma/admin/need_reboot`.
</details>
......@@ -182,6 +190,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- **Breaking**: Using third party engines for user recommendation
<details>
<summary>API Changes</summary>
- **Breaking**: AdminAPI: migrate_from_db endpoint
</details>
......
......@@ -173,7 +173,7 @@
"application/ld+json" => ["activity+json"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
config :tesla, adapter: Tesla.Adapter.Gun
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
......@@ -555,6 +555,7 @@
attachments_cleanup: 5,
new_users_digest: 1
],
plugins: [Oban.Plugins.Pruner],
crontab: [
{"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
{"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
......@@ -690,32 +691,30 @@
prepare: :unnamed
config :pleroma, :connections_pool,
checkin_timeout: 250,
reclaim_multiplier: 0.1,
connection_acquisition_wait: 250,
connection_acquisition_retries: 5,
max_connections: 250,
retry: 1,
retry_timeout: 1000,
max_idle_time: 30_000,
retry: 0,
await_up_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_overflow: 10,
timeout: 150_000
max_waiting: 10
],
media: [
size: 50,
max_overflow: 10,
timeout: 150_000
max_waiting: 10
],
upload: [
size: 25,
max_overflow: 5,
timeout: 300_000
max_waiting: 5
],
default: [
size: 10,
max_overflow: 2,
timeout: 10_000
max_waiting: 2
]
config :pleroma, :hackney_pools,
......
......@@ -1779,8 +1779,8 @@
%{
key: :whitelist,
type: {:list, :string},
description: "List of domains to bypass the mediaproxy",
suggestions: ["example.com"]
description: "List of hosts with scheme to bypass the mediaproxy",
suggestions: ["http://example.com"]
}
]
},
......@@ -2019,13 +2019,15 @@
label: "Pleroma Admin Token",
type: :group,
description:
"Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the `admin_token` parameter",
"Allows setting a token that can be used to authenticate requests with admin privileges without a normal user account token. Append the `admin_token` parameter to requests to utilize it. (Please reconsider using HTTP Basic Auth or OAuth-based authentication if possible)",
children: [
%{
key: :admin_token,
type: :string,
description: "Admin token",
suggestions: ["We recommend a secure random string or UUID"]
suggestions: [
"Please use a high entropy string or UUID"
]
}
]
},
......@@ -3170,36 +3172,37 @@
description: "Advanced settings for `gun` connections pool",
children: [
%{
key: :checkin_timeout,
type: :integer,
description: "Timeout to checkin connection from pool. Default: 250ms.",
suggestions: [250]
},
%{
key: :max_connections,
key: :connection_acquisition_wait,
type: :integer,
description: "Maximum number of connections in the pool. Default: 250 connections.",
description:
"Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. Default: 250ms.",
suggestions: [250]
},
%{
key: :retry,
key: :connection_acquisition_retries,
type: :integer,
description:
"Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
suggestions: [1]
"Number of attempts to acquire the connection from the pool if it is overloaded. Default: 5",
suggestions: [5]
},
%{
key: :retry_timeout,
key: :max_connections,
type: :integer,
description:
"Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
suggestions: [1000]
description: "Maximum number of connections in the pool. Default: 250 connections.",
suggestions: [250]
},
%{
key: :await_up_timeout,
type: :integer,
description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
suggestions: [5000]
},
%{
key: :reclaim_multiplier,
type: :integer,
description:
"Multiplier for the number of idle connection to be reclaimed if the pool is full. For example if the pool maxes out at 250 connections and this setting is set to 0.3, the pool will reclaim at most 75 idle connections if it's overloaded. Default: 0.1",
suggestions: [0.1]
}
]
},
......@@ -3208,108 +3211,29 @@
key: :pools,
type: :group,
description: "Advanced settings for `gun` workers pools",
children: [
%{
key: :federation,
type: :keyword,
description: "Settings for federation pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :media,
type: :keyword,
description: "Settings for media pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :upload,
type: :keyword,
description: "Settings for upload pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [25]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [5]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [300_000]
}
]
},
%{
key: :default,
type: :keyword,
description: "Settings for default pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [10]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [2]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [10_000]
}
]
}
]
children:
Enum.map([:federation, :media, :upload, :default], fn pool_name ->
%{
key: pool_name,
type: :keyword,
description: "Settings for #{pool_name} pool.",
children: [
%{
key: :size,
type: :integer,
description: "Maximum number of concurrent requests in the pool.",
suggestions: [50]
},
%{
key: :max_waiting,
type: :integer,
description:
"Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made",
suggestions: [10]
}
]
}
end)
},
%{
group: :pleroma,
......
......@@ -129,6 +129,11 @@
config :pleroma, :instances_favicons, enabled: true
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
streaming_enabled: true,
public_endpoint: nil
if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs"
else
......
......@@ -236,6 +236,7 @@ Has theses additional parameters (which are the same as in Pleroma-API):
- `pleroma.metadata.features`: A list of supported features
- `pleroma.metadata.federation`: The federation restrictions of this instance
- `pleroma.metadata.fields_limits`: A list of values detailing the length and count limitation for various instance-configurable fields.
- `pleroma.metadata.post_formats`: A list of the allowed post format types
- `vapid_public_key`: The public key needed for push messages
## Markers
......
......@@ -301,11 +301,8 @@ See [Admin-API](admin_api.md)
* Method `PUT`
* Authentication: required
* Params:
* `followers`: BOOLEAN field, receives notifications from followers
* `follows`: BOOLEAN field, receives notifications from people the user follows
* `remote`: BOOLEAN field, receives notifications from people on remote instances
* `local`: BOOLEAN field, receives notifications from people on the local instance
* `privacy_option`: BOOLEAN field. When set to true, it removes the contents of a message from the push notification.
* `block_from_strangers`: BOOLEAN field, blocks notifications from accounts you do not follow
* `hide_notification_contents`: BOOLEAN field. When set to true, it removes the contents of a message from the push notification.
* Response: JSON. Returns `{"status": "success"}` if the update was successful, otherwise returns `{"error": "error_msg"}`
## `/api/pleroma/healthcheck`
......
......@@ -268,6 +268,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
* `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`).
## :emoji
* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]`
* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
......@@ -276,13 +277,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
memory for this amount of seconds multiplied by the number of files.
## :media_proxy
* `enabled`: Enables proxying of remote media to the instance’s proxy
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
* `whitelist`: List of domains to bypass the mediaproxy
* `whitelist`: List of hosts with scheme to bypass the mediaproxy (e.g. `https://example.com`)
* `invalidation`: options for remove media from cache after delete object:
* `enabled`: Enables purge cache
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
* `enabled`: Enables purge cache
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
### Purge cache strategy
......@@ -294,6 +296,7 @@ Urls of attachments pass to script as arguments.
* `script_path`: path to external script.
Example:
```elixir
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
script_path: "./installation/nginx-cache-purge.example"
......@@ -461,36 +464,32 @@ For each pool, the options are:
*For `gun` adapter*
Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
Settings for HTTP connection pool.
For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000.
It will increase memory usage, but federation would work faster.
* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms.
* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.
* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.
* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
* `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries.
* `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart.
* `:max_connections` - Maximum number of connections in the pool.
* `:await_up_timeout` - Timeout to connect to the host.
* `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded.
### :pools
*For `gun` adapter*
Advanced settings for workers pools.
Settings for request pools. These pools are limited on top of `:connections_pool`.
There are four pools used:
* `:federation` for the federation jobs.
You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
* `:media` for rich media, media proxy
* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
* `:default` for other requests
* `:federation` for the federation jobs. You may want this pool's max_connections to be at least equal to the number of federator jobs + retry queue jobs.
* `:media` - for rich media, media proxy.
* `:upload` - for proxying media when a remote uploader is used and `proxy_remote: true`.
* `:default` - for other requests.
For each pool, the options are:
* `:size` - how much workers the pool can hold
* `:size` - limit to how much requests can be concurrently executed.
* `:timeout` - timeout while `gun` will wait for response
* `:max_overflow` - additional workers if pool is under load
* `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped.
## Captcha
......@@ -645,8 +644,7 @@ Email notifications settings.
Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage):
* `repo` - app's Ecto repo (`Pleroma.Repo`)
* `verbose` - logs verbosity
* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`)
* `log` - logs verbosity
* `queues` - job queues (see below)
* `crontab` - periodic jobs, see [`Oban.Cron`](#obancron)
......@@ -831,6 +829,8 @@ or
curl -H "X-Admin-Token: somerandomtoken" "http://localhost:4000/api/pleroma/admin/users/invites"
```
Warning: it's discouraged to use this feature because of the associated security risk: static / rarely changed instance-wide token is much weaker compared to email-password pair of a real admin user; consider using HTTP Basic Auth or OAuth-based authentication instead.
### :auth
* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
......
......@@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
@moduledoc """
Example:
> mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588" # set false only for parallel588 user
> mix pleroma.notification_settings --privacy-option=true # set true for all users
> mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588" # set false only for parallel588 user
> mix pleroma.notification_settings --hide-notification-contents=true # set true for all users
"""
......@@ -19,16 +19,16 @@ def run(args) do
OptionParser.parse(
args,
strict: [
privacy_option: :boolean,
hide_notification_contents: :boolean,
email_users: :string,
nickname_users: :string
]
)
privacy_option = Keyword.get(options, :privacy_option)
hide_notification_contents = Keyword.get(options, :hide_notification_contents)
if not is_nil(privacy_option) do
privacy_option
if not is_nil(hide_notification_contents) do
hide_notification_contents
|> build_query(options)
|> Pleroma.Repo.update_all([])
end
......@@ -36,15 +36,15 @@ def run(args) do
shell_info("Done")
end
defp build_query(privacy_option, options) do
defp build_query(hide_notification_contents, options) do
query =
from(u in Pleroma.User,
update: [
set: [
notification_settings:
fragment(
"jsonb_set(notification_settings, '{privacy_option}', ?)",
^privacy_option
"jsonb_set(notification_settings, '{hide_notification_contents}', ?)",
^hide_notification_contents
)
]
]
......
......@@ -35,6 +35,11 @@ def user_agent do
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
# Scrubbers are compiled at runtime and therefore will cause a conflict
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
Config.DeprecationWarnings.warn()
......@@ -220,9 +225,7 @@ defp task_children(_) do
# start hackney and gun pools in tests
defp http_children(_, :test) do
hackney_options = Config.get([:hackney_pools, :federation])
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
[hackney_pool, Pleroma.Pool.Supervisor]
http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil)
end
defp http_children(Tesla.Adapter.Hackney, _) do
......@@ -241,7 +244,10 @@ defp http_children(Tesla.Adapter.Hackney, _) do
end
end
defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
defp http_children(Tesla.Adapter.Gun, _) do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
defp http_children(_, _), do: []
......
......@@ -54,6 +54,7 @@ def warn do
check_hellthread_threshold()
mrf_user_allowlist()
check_old_mrf_config()
check_media_proxy_whitelist_config()
end
def check_old_mrf_config do
......@@ -65,7 +66,7 @@ def check_old_mrf_config do
move_namespace_and_warn(@mrf_config_map, warning_preface)
end