Commit 49444981 authored by minibikini's avatar minibikini
Browse files

Merge branch 'develop' into feature/compat/push-subscriptions

# Conflicts:
#	lib/pleroma/application.ex
#	lib/pleroma/plugs/oauth_plug.ex
parents bac58b15 ccf0b46d
......@@ -6,6 +6,9 @@
/uploads
/test/uploads
/.elixir_ls
/test/fixtures/test_tmp.txt
/test/fixtures/image_tmp.jpg
/doc
# Prevent committing custom emojis
/priv/static/emoji/custom/*
......@@ -28,4 +31,4 @@ erl_crash.dump
.env
# Editor config
/.vscode
\ No newline at end of file
/.vscode
......@@ -10,18 +10,18 @@
config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
strip_exif: false
filters: [],
proxy_remote: false,
proxy_opts: []
config :pleroma, Pleroma.Uploaders.Local,
uploads: "uploads",
uploads_url: "{{base_url}}/media/{{file}}"
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
public_endpoint: "https://s3.amazonaws.com",
force_media_proxy: false
public_endpoint: "https://s3.amazonaws.com"
config :pleroma, Pleroma.Uploaders.MDII,
cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
......@@ -72,6 +72,7 @@
config :pleroma, :websub, Pleroma.Web.Websub
config :pleroma, :ostatus, Pleroma.Web.OStatus
config :pleroma, :httpoison, Pleroma.HTTP
config :tesla, adapter: Tesla.Adapter.Hackney
# Configures http settings, upstream proxy etc.
config :pleroma, :http, proxy_url: nil
......@@ -150,9 +151,11 @@
config :pleroma, :media_proxy,
enabled: false,
redirect_on_failure: true
# base_url: "https://cache.pleroma.social"
# base_url: "https://cache.pleroma.social",
proxy_opts: [
# inline_content_types: [] | false | true,
# http: [:insecure]
]
config :pleroma, :chat, enabled: true
......
......@@ -5,11 +5,19 @@ If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherw
## Pleroma.Upload
* `uploader`: Select which `Pleroma.Uploaders` to use
* `strip_exif`: boolean, uses ImageMagick(!) to strip exif.
* `filters`: List of `Pleroma.Upload.Filter` to use.
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
## Pleroma.Uploaders.Local
* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory
* `uploads_url`: The URL to access a user-uploaded file, ``{{base_url}}`` is replaced to the instance URL and ``{{file}}`` to the filename. Useful when you want to proxy the media files via another host.
## Pleroma.Upload.Filter.Mogrify
* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", {"impode", "1"}]`.
## :uri_schemes
* `valid_schemes`: List of the scheme part that is considered valid to be an URL
......@@ -68,7 +76,8 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
## :media_proxy
* `enabled`: Enables proxying of remote media to the instance’s proxy
* `redirect_on_failure`: Use the original URL when Media Proxy fails to get it
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
## :gopher
* `enabled`: Enables the gopher interface
......
......@@ -49,11 +49,10 @@
hostname: "localhost",
pool_size: 10
try do
if File.exists?("./config/dev.secret.exs") do
import_config "dev.secret.exs"
rescue
_ ->
IO.puts(
"!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
)
else
IO.puts(
"!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
)
end
......@@ -9,7 +9,7 @@
# Print only warnings and errors during test
config :logger, level: :warn
config :pleroma, Pleroma.Upload, uploads: "test/uploads"
config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
# Configure your database
config :pleroma, Pleroma.Repo,
......@@ -25,7 +25,7 @@
config :pleroma, :websub, Pleroma.Web.WebsubMock
config :pleroma, :ostatus, Pleroma.Web.OStatusMock
config :pleroma, :httpoison, HTTPoisonMock
config :tesla, adapter: Tesla.Mock
try do
import_config "test.secret.exs"
......
......@@ -70,10 +70,12 @@ server {
client_max_body_size 16m;
}
location /proxy {
location ~ ^/(media|proxy) {
proxy_cache pleroma_media_cache;
proxy_cache_lock on;
proxy_ignore_client_abort on;
proxy_buffering off;
chunked_transfer_encoding on;
proxy_pass http://localhost:4000;
}
}
......@@ -8,7 +8,7 @@ defmodule Mix.Tasks.SetModerator do
"""
use Mix.Task
import Mix.Ecto
import Ecto.Changeset
alias Pleroma.{Repo, User}
def run([nickname | rest]) do
......@@ -21,14 +21,15 @@ def run([nickname | rest]) do
end
with %User{local: true} = user <- User.get_by_nickname(nickname) do
info =
user.info
|> Map.put("is_moderator", !!moderator)
info_cng = User.Info.admin_api_update(user.info, %{is_moderator: !!moderator})
cng = User.info_changeset(user, %{info: info})
{:ok, user} = User.update_and_set_cache(cng)
user_cng =
Ecto.Changeset.change(user)
|> put_embed(:info, info_cng)
IO.puts("Moderator status of #{nickname}: #{user.info["is_moderator"]}")
{:ok, user} = User.update_and_set_cache(user_cng)
IO.puts("Moderator status of #{nickname}: #{user.info.is_moderator}")
else
_ ->
IO.puts("No local user #{nickname}")
......
defmodule Mix.Tasks.MigrateLocalUploads do
use Mix.Task
import Mix.Ecto
alias Pleroma.{Upload, Uploaders.Local, Uploaders.S3}
require Logger
@log_every 50
@shortdoc "Migrate uploads from local to remote storage"
def run([target_uploader | args]) do
delete? = Enum.member?(args, "--delete")
Application.ensure_all_started(:pleroma)
local_path = Pleroma.Config.get!([Local, :uploads])
uploader = Module.concat(Pleroma.Uploaders, target_uploader)
unless Code.ensure_loaded?(uploader) do
raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
end
target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
unless target_enabled? do
Pleroma.Config.put([Upload, :uploader], uploader)
end
Logger.info("Migrating files from local #{local_path} to #{to_string(uploader)}")
if delete? do
Logger.warn(
"Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
)
:timer.sleep(:timer.seconds(5))
end
uploads =
File.ls!(local_path)
|> Enum.map(fn id ->
root_path = Path.join(local_path, id)
cond do
File.dir?(root_path) ->
files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
case List.first(files) do
{id, file, path} ->
{%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
root_path}
_ ->
nil
end
File.exists?(root_path) ->
file = Path.basename(id)
[hash, ext] = String.split(id, ".")
{%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
true ->
nil
end
end)
|> Enum.filter(& &1)
total_count = length(uploads)
Logger.info("Found #{total_count} uploads")
uploads
|> Task.async_stream(
fn {upload, root_path} ->
case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
{:ok, _} ->
if delete?, do: File.rm_rf!(root_path)
Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
:ok
error ->
Logger.error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
end
end,
timeout: 150_000
)
|> Stream.chunk_every(@log_every)
|> Enum.reduce(0, fn done, count ->
count = count + length(done)
Logger.info("Uploaded #{count}/#{total_count} files")
count
end)
Logger.info("Done!")
end
def run(_) do
Logger.error("Usage: migrate_local_uploads S3|Swift [--delete]")
end
end
......@@ -4,3 +4,4 @@ CREATE DATABASE pleroma_dev OWNER pleroma;
--Extensions made by ecto.migrate that need superuser access
CREATE EXTENSION IF NOT EXISTS citext;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
defmodule Mix.Tasks.SetAdmin do
use Mix.Task
import Ecto.Changeset
alias Pleroma.User
@doc """
......@@ -9,21 +10,22 @@ defmodule Mix.Tasks.SetAdmin do
def run([nickname | rest]) do
Application.ensure_all_started(:pleroma)
status =
admin =
case rest do
[status] -> status == "true"
[admin] -> admin == "true"
_ -> true
end
with %User{local: true} = user <- User.get_by_nickname(nickname) do
info =
user.info
|> Map.put("is_admin", !!status)
info_cng = User.Info.admin_api_update(user.info, %{is_admin: !!admin})
cng = User.info_changeset(user, %{info: info})
{:ok, user} = User.update_and_set_cache(cng)
user_cng =
Ecto.Changeset.change(user)
|> put_embed(:info, info_cng)
IO.puts("Admin status of #{nickname}: #{user.info["is_admin"]}")
{:ok, user} = User.update_and_set_cache(user_cng)
IO.puts("Admin status of #{nickname}: #{user.info.is_admin}")
else
_ ->
IO.puts("No local user #{nickname}")
......
......@@ -10,11 +10,11 @@ defmodule Mix.Tasks.SetLocked do
"""
use Mix.Task
import Mix.Ecto
import Ecto.Changeset
alias Pleroma.{Repo, User}
def run([nickname | rest]) do
ensure_started(Repo, [])
Application.ensure_all_started(:pleroma)
locked =
case rest do
......@@ -23,14 +23,15 @@ def run([nickname | rest]) do
end
with %User{local: true} = user <- User.get_by_nickname(nickname) do
info =
user.info
|> Map.put("locked", !!locked)
info_cng = User.Info.profile_update(user.info, %{locked: !!locked})
cng = User.info_changeset(user, %{info: info})
user = Repo.update!(cng)
user_cng =
Ecto.Changeset.change(user)
|> put_embed(:info, info_cng)
IO.puts("locked status of #{nickname}: #{user.info["locked"]}")
{:ok, user} = User.update_and_set_cache(user_cng)
IO.puts("Locked status of #{nickname}: #{user.info.locked}")
else
_ ->
IO.puts("No local user #{nickname}")
......
defmodule Pleroma.Application do
use Application
import Supervisor.Spec
@name "Pleroma"
@version Mix.Project.config()[:version]
......@@ -7,11 +8,15 @@ def name, do: @name
def version, do: @version
def named_version(), do: @name <> " " <> @version
def user_agent() do
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
end
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
@env Mix.env()
def start(_type, _args) do
import Supervisor.Spec
import Cachex.Spec
# Define workers and child supervisors to be supervised
......@@ -20,10 +25,6 @@ def start(_type, _args) do
# Start the Ecto repository
supervisor(Pleroma.Repo, []),
worker(Pleroma.Emoji, []),
# Start the endpoint when the application starts
supervisor(Pleroma.Web.Endpoint, []),
# Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
# worker(Pleroma.Worker, [arg1, arg2, arg3]),
worker(
Cachex,
[
......@@ -63,21 +64,18 @@ def start(_type, _args) do
],
id: :cachex_idem
),
worker(Pleroma.Web.Federator, []),
worker(Pleroma.Web.Federator.RetryQueue, []),
worker(Pleroma.Gopher.Server, []),
worker(Pleroma.Web.Federator, []),
worker(Pleroma.Stats, []),
worker(Pleroma.Web.Push, [])
] ++
if @env == :test,
do: [],
else:
[worker(Pleroma.Web.Streamer, [])] ++
if(
!chat_enabled(),
do: [],
else: [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
)
streamer_child() ++
chat_child() ++
[
# Start the endpoint when the application starts
supervisor(Pleroma.Web.Endpoint, []),
worker(Pleroma.Gopher.Server, [])
]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
......@@ -85,7 +83,20 @@ def start(_type, _args) do
Supervisor.start_link(children, opts)
end
defp chat_enabled do
Application.get_env(:pleroma, :chat, []) |> Keyword.get(:enabled)
if Mix.env() == :test do
defp streamer_child(), do: []
defp chat_child(), do: []
else
defp streamer_child() do
[worker(Pleroma.Web.Streamer, [])]
end
defp chat_child() do
if Pleroma.Config.get([:chat, :enabled]) do
[worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
else
[]
end
end
end
end
......@@ -39,4 +39,18 @@ def put([parent_key | keys], value) do
def put(key, value) do
Application.put_env(:pleroma, key, value)
end
def delete([key]), do: delete(key)
def delete([parent_key | keys]) do
{_, parent} =
Application.get_env(:pleroma, parent_key)
|> get_and_update_in(keys, fn _ -> :pop end)
Application.put_env(:pleroma, parent_key, parent)
end
def delete(key) do
Application.delete_env(:pleroma, key)
end
end
......@@ -114,10 +114,10 @@ def add_user_links({subs, text}, mentions) do
subs =
subs ++
Enum.map(mentions, fn {match, %User{ap_id: ap_id, info: info}, uuid} ->
Enum.map(mentions, fn {match, %User{id: id, ap_id: ap_id, info: info}, uuid} ->
ap_id =
if is_binary(info["source_data"]["url"]) do
info["source_data"]["url"]
if is_binary(info.source_data["url"]) do
info.source_data["url"]
else
ap_id
end
......@@ -125,7 +125,7 @@ def add_user_links({subs, text}, mentions) do
short_match = String.split(match, "@") |> tl() |> hd()
{uuid,
"<span><a class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
"<span><a data-user='#{id}' class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
end)
{subs, uuid_text}
......@@ -147,7 +147,11 @@ def add_hashtag_links({subs, text}, tags) do
subs =
subs ++
Enum.map(tags, fn {tag_text, tag, uuid} ->
url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{tag_text}</a>"
url =
"<a data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
tag_text
}</a>"
{uuid, url}
end)
......
......@@ -6,28 +6,29 @@ def start_link() do
config = Pleroma.Config.get(:gopher, [])
ip = Keyword.get(config, :ip, {0, 0, 0, 0})
port = Keyword.get(config, :port, 1234)
GenServer.start_link(__MODULE__, [ip, port], [])
end
def init([ip, port]) do
if Pleroma.Config.get([:gopher, :enabled], false) do
Logger.info("Starting gopher server on #{port}")
:ranch.start_listener(
:gopher,
100,
:ranch_tcp,
[port: port],
__MODULE__.ProtocolHandler,
[]
)
{:ok, %{ip: ip, port: port}}
if Keyword.get(config, :enabled, false) do
GenServer.start_link(__MODULE__, [ip, port], [])
else
Logger.info("Gopher server disabled")
{:ok, nil}
:ignore
end
end
def init([ip, port]) do
Logger.info("Starting gopher server on #{port}")
:ranch.start_listener(
:gopher,
100,
:ranch_tcp,
[port: port],
__MODULE__.ProtocolHandler,
[]
)
{:ok, %{ip: ip, port: port}}
end
end
defmodule Pleroma.Gopher.Server.ProtocolHandler do
......
......@@ -45,7 +45,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
Meta.strip_comments()
# links
Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes)
Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
Meta.allow_tag_with_these_attributes("a", ["name", "title"])
# paragraphs and linebreaks
......@@ -86,7 +86,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.remove_cdata_sections_before_scrub()
Meta.strip_comments()
Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes)
Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
Meta.allow_tag_with_these_attributes("a", ["name", "title"])
Meta.allow_tag_with_these_attributes("abbr", ["title"])
......
defmodule Pleroma.HTTP.Connection do
@moduledoc """
Connection for http-requests.
"""
@hackney_options [pool: :default]
@adapter Application.get_env(:tesla, :adapter)
@doc """
Configure a client connection
# Returns
Tesla.Env.client
"""
@spec new(Keyword.t()) :: Tesla.Env.client()
def new(opts \\ []) do
Tesla.client([], {@adapter, hackney_options(opts)})
end
# fetch Hackney options
#
defp hackney_options(opts \\ []) do
options = Keyword.get(opts, :adapter, [])
@hackney_options ++ options
end
end
defmodule Pleroma.HTTP do
require HTTPoison
@moduledoc """
"""
alias Pleroma.HTTP.Connection
alias Pleroma.HTTP.RequestBuilder, as: Builder
@doc """
Builds and perform http request.
# Arguments:
`method` - :get, :post, :put, :delete
`url`
`body`
`headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
`options` - custom, per-request middleware or adapter options
# Returns: