Commit 339c3d4d authored by Ivan Tashkinov's avatar Ivan Tashkinov

Merge remote-tracking branch 'remotes/pleroma/develop' into merge/develop-20200328

# Conflicts:
#	lib/pleroma/application.ex
#	lib/pleroma/web/metadata/opengraph.ex
parents 7c9c9dd1 7879369e
......@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added
- NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
- Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
<details>
<summary>API Changes</summary>
- Mastodon API: Support for `include_types` in `/api/v1/notifications`.
......
......@@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make &&\
mkdir release &&\
mix release --path release
FROM alpine:3.9
FROM alpine:3.11
ARG BUILD_DATE
ARG VCS_REF
......@@ -33,7 +33,7 @@ ARG DATA=/var/lib/pleroma
RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
apk update &&\
apk add ncurses postgresql-client &&\
apk add imagemagick ncurses postgresql-client &&\
adduser --system --shell /bin/false --home ${HOME} pleroma &&\
mkdir -p ${DATA}/uploads &&\
mkdir -p ${DATA}/static &&\
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
defmodule Pleroma.LoadTesting.Helper do
defmacro __using__(_) do
quote do
import Ecto.Query
alias Pleroma.Repo
alias Pleroma.User
alias Ecto.Adapters.SQL
alias Pleroma.Repo
defp to_sec(microseconds), do: microseconds / 1_000_000
end
def to_sec(microseconds), do: microseconds / 1_000_000
def clean_tables do
IO.puts("Deleting old data...\n")
SQL.query!(Repo, "TRUNCATE users CASCADE;")
SQL.query!(Repo, "TRUNCATE activities CASCADE;")
SQL.query!(Repo, "TRUNCATE objects CASCADE;")
SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;")
end
end
defmodule Pleroma.LoadTesting.Users do
@moduledoc """
Module for generating users with friends.
"""
import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [to_sec: 1]
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.User.Query
@defaults [
users: 20_000,
friends: 100
]
@max_concurrency 10
@spec generate(keyword()) :: User.t()
def generate(opts \\ []) do
opts = Keyword.merge(@defaults, opts)
generate_users(opts[:users])
main_user =
Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1))
make_friends(main_user, opts[:friends])
Repo.get(User, main_user.id)
end
def generate_users(max) do
IO.puts("Starting generating #{max} users...")
{time, users} =
:timer.tc(fn ->
Task.async_stream(
1..max,
&generate_user(&1),
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Enum.to_list()
end)
IO.puts("Generating users took #{to_sec(time)} sec.\n")
users
end
defp generate_user(i) do
remote = Enum.random([true, false])
%User{
name: "Test テスト User #{i}",
email: "user#{i}@example.com",
nickname: "nick#{i}",
password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
bio: "Tester Number #{i}",
local: !remote
}
|> user_urls()
|> Repo.insert!()
end
defp user_urls(%{local: true} = user) do
urls = %{
ap_id: User.ap_id(user),
follower_address: User.ap_followers(user),
following_address: User.ap_following(user)
}
Map.merge(user, urls)
end
defp user_urls(%{local: false} = user) do
base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"])
ap_id = "https://#{base_domain}/users/#{user.nickname}"
urls = %{
ap_id: ap_id,
follower_address: ap_id <> "/followers",
following_address: ap_id <> "/following"
}
Map.merge(user, urls)
end
def make_friends(main_user, max) when is_integer(max) do
IO.puts("Starting making friends for #{max} users...")
{time, _} =
:timer.tc(fn ->
number_of_users =
(max / 2)
|> Kernel.trunc()
main_user
|> get_users(%{limit: number_of_users, local: :local})
|> run_stream(main_user)
main_user
|> get_users(%{limit: number_of_users, local: :external})
|> run_stream(main_user)
end)
IO.puts("Making friends took #{to_sec(time)} sec.\n")
end
def make_friends(%User{} = main_user, %User{} = user) do
{:ok, _} = User.follow(main_user, user)
{:ok, _} = User.follow(user, main_user)
end
@spec get_users(User.t(), keyword()) :: [User.t()]
def get_users(user, opts) do
criteria = %{limit: opts[:limit]}
criteria =
if opts[:local] do
Map.put(criteria, opts[:local], true)
else
criteria
end
criteria =
if opts[:friends?] do
Map.put(criteria, :friends, user)
else
criteria
end
query =
criteria
|> Query.build()
|> random_without_user(user)
query =
if opts[:friends?] == false do
friends_ids =
%{friends: user}
|> Query.build()
|> Repo.all()
|> Enum.map(& &1.id)
from(u in query, where: u.id not in ^friends_ids)
else
query
end
Repo.all(query)
end
defp random_without_user(query, user) do
from(u in query,
where: u.id != ^user.id,
order_by: fragment("RANDOM()")
)
end
defp run_stream(users, main_user) do
Task.async_stream(users, &make_friends(main_user, &1),
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Stream.run()
end
end
defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
import Ecto.Query
alias Pleroma.Repo
alias Pleroma.Web.MastodonAPI.TimelineController
def run(_args) do
Mix.Pleroma.start_pleroma()
activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
......@@ -11,8 +14,8 @@ def run(_args) do
if activities_count == 0 do
IO.puts("Did not find any activities, cleaning and generating")
clean_tables()
Generator.generate_users(users_max: 10)
Generator.generate_tagged_activities()
Pleroma.LoadTesting.Users.generate_users(10)
Pleroma.LoadTesting.Activities.generate_tagged_activities()
else
IO.puts("Found #{activities_count} activities, won't generate new ones")
end
......@@ -34,7 +37,7 @@ def run(_args) do
Benchee.run(
%{
"Hashtag fetching, any" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"any" => tags
},
......@@ -44,7 +47,7 @@ def run(_args) do
end,
# Will always return zero results because no overlapping hashtags are generated.
"Hashtag fetching, all" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"all" => tags
},
......@@ -64,7 +67,7 @@ def run(_args) do
Benchee.run(
%{
"Hashtag fetching" => fn tag ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"tag" => tag
},
......@@ -77,11 +80,4 @@ def run(_args) do
time: 5
)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end
defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
alias Pleroma.Web.CommonAPI
alias Plug.Conn
def run(_args) do
Mix.Pleroma.start_pleroma()
......@@ -11,7 +12,7 @@ def run(_args) do
# Cleaning tables
clean_tables()
[{:ok, user} | users] = Generator.generate_users(users_max: 1000)
[{:ok, user} | users] = Pleroma.LoadTesting.Users.generate_users(1000)
# Let the user make 100 posts
......@@ -38,8 +39,8 @@ def run(_args) do
"user timeline, no followers" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Plug.Conn.assign(:user, reading_user)
|> Plug.Conn.assign(:skip_link_headers, true)
|> Conn.assign(:user, reading_user)
|> Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
......@@ -56,8 +57,8 @@ def run(_args) do
"user timeline, all following" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Plug.Conn.assign(:user, reading_user)
|> Plug.Conn.assign(:skip_link_headers, true)
|> Conn.assign(:user, reading_user)
|> Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
......@@ -66,11 +67,4 @@ def run(_args) do
time: 60
)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end
defmodule Mix.Tasks.Pleroma.LoadTesting do
use Mix.Task
use Pleroma.LoadTesting.Helper
import Mix.Pleroma
import Pleroma.LoadTesting.Generator
import Pleroma.LoadTesting.Fetcher
import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
alias Pleroma.Repo
alias Pleroma.User
@shortdoc "Factory for generation data"
@moduledoc """
Generates data like:
- local/remote users
- local/remote activities with notifications
- direct messages
- long thread
- non visible posts
- local/remote activities with differrent visibility:
- simple activiities
- with emoji
- with mentions
- hellthreads
- with attachments
- with tags
- likes
- reblogs
- simple threads
- long threads
## Generate data
MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --friends 1000 --iterations 170 --friends_used 20 --non_friends_used 20
MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -f 1000 -i 170 -fu 20 -nfu 20
Options:
- `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
- `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
- `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
- `--friends NUMBER` - number of friends for main user. Defaults to: 1000. Alias: `-f`
- `--iterations NUMBER` - number of iterations to generate activities. For each iteration in database is inserted about 120+ activities with different visibility, actors and types.Defaults to: 170. Alias: `-i`
- `--friends_used NUMBER` - number of main user friends used in activity generation. Defaults to: 20. Alias: `-fu`
- `--non_friends_used NUMBER` - number of non friends used in activity generation. Defaults to: 20. Alias: `-nfu`
"""
@aliases [u: :users, d: :dms, t: :thread_length]
@aliases [u: :users, f: :friends, i: :iterations, fu: :friends_used, nfu: :non_friends_used]
@switches [
users: :integer,
dms: :integer,
thread_length: :integer
friends: :integer,
iterations: :integer,
friends_used: :integer,
non_friends_used: :integer
]
@users_default 20_000
@dms_default 1_000
@thread_length_default 2_000
def run(args) do
start_pleroma()
Pleroma.Config.put([:instance, :skip_thread_containment], true)
{opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
users_max = Keyword.get(opts, :users, @users_default)
dms_max = Keyword.get(opts, :dms, @dms_default)
thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
Mix.Pleroma.start_pleroma()
clean_tables()
{opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
opts =
Keyword.put(opts, :users_max, users_max)
|> Keyword.put(:dms_max, dms_max)
|> Keyword.put(:thread_length, thread_length)
generate_users(opts)
# main user for queries
IO.puts("Fetching local main user...")
{time, user} =
:timer.tc(fn ->
Repo.one(
from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
)
end)
IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
IO.puts("Fetching local users...")
{time, users} =
:timer.tc(fn ->
Repo.all(
from(u in User,
where: u.id != ^user.id,
where: u.local == true,
order_by: fragment("RANDOM()"),
limit: 10
)
)
end)
IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
IO.puts("Fetching remote users...")
{time, remote_users} =
:timer.tc(fn ->
Repo.all(
from(u in User,
where: u.id != ^user.id,
where: u.local == false,
order_by: fragment("RANDOM()"),
limit: 10
)
)
end)
IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
generate_activities(user, users)
generate_remote_activities(user, remote_users)
generate_like_activities(
user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create"))
)
generate_dms(user, users, opts)
{:ok, activity} = generate_long_thread(user, users, opts)
generate_non_visible_message(user, users)
user = Pleroma.LoadTesting.Users.generate(opts)
Pleroma.LoadTesting.Activities.generate(user, opts)
IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
......@@ -120,19 +61,6 @@ def run(args) do
"Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
)
fetch_user(user)
query_timelines(user)
query_notifications(user)
query_dms(user)
query_long_thread(user, activity)
Pleroma.Config.put([:instance, :skip_thread_containment], false)
query_timelines(user)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
Pleroma.LoadTesting.Fetcher.run_benchmarks(user)
end
end
......@@ -39,7 +39,7 @@
adapter: Ecto.Adapters.Postgres,
username: "postgres",
password: "postgres",
database: "pleroma_test",
database: "pleroma_benchmark",
hostname: System.get_env("DB_HOST") || "localhost",
pool_size: 10
......
......@@ -58,20 +58,6 @@
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
......@@ -185,21 +171,13 @@
"application/ld+json" => ["activity+json"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
config :tesla, adapter: Tesla.Adapter.Gun
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
proxy_url: nil,
send_user_agent: true,
user_agent: :default,
adapter: [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
]
]
adapter: []
config :pleroma, :instance,
name: "Pleroma",
......@@ -667,6 +645,49 @@
parameters: [gin_fuzzy_search_limit: "500"],
prepare: :unnamed
config :pleroma, :connections_pool,
checkin_timeout: 250,
max_connections: 250,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_overflow: 10,
timeout: 150_000
],