Verified Commit b19597f6 authored by href's avatar href
Browse files

reverse proxy / uploads

parent 52ce3685
......@@ -6,6 +6,9 @@
/uploads
/test/uploads
/.elixir_ls
/test/fixtures/test_tmp.txt
/test/fixtures/image_tmp.jpg
/doc
# Prevent committing custom emojis
/priv/static/emoji/custom/*
......@@ -28,4 +31,4 @@ erl_crash.dump
.env
# Editor config
/.vscode
\ No newline at end of file
/.vscode
......@@ -12,16 +12,15 @@
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
strip_exif: false
strip_exif: false,
proxy_remote: false,
proxy_opts: [inline_content_types: true, keep_user_agent: true]
config :pleroma, Pleroma.Uploaders.Local,
uploads: "uploads",
uploads_url: "{{base_url}}/media/{{file}}"
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
public_endpoint: "https://s3.amazonaws.com",
force_media_proxy: false
public_endpoint: "https://s3.amazonaws.com"
config :pleroma, Pleroma.Uploaders.MDII,
cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
......@@ -150,9 +149,11 @@
config :pleroma, :media_proxy,
enabled: false,
redirect_on_failure: true
# base_url: "https://cache.pleroma.social"
# base_url: "https://cache.pleroma.social",
proxy_opts: [
# inline_content_types: [] | false | true,
# http: [:insecure]
]
config :pleroma, :chat, enabled: true
......
......@@ -9,7 +9,7 @@
# Print only warnings and errors during test
config :logger, level: :warn
config :pleroma, Pleroma.Upload, uploads: "test/uploads"
config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
# Configure your database
config :pleroma, Pleroma.Repo,
......
defmodule Mix.Tasks.MigrateLocalUploads do
use Mix.Task
import Mix.Ecto
alias Pleroma.{Upload, Uploaders.Local, Uploaders.S3}
require Logger
@log_every 50
@shortdoc "Migrate uploads from local to remote storage"
def run([target_uploader | args]) do
delete? = Enum.member?(args, "--delete")
Application.ensure_all_started(:pleroma)
local_path = Pleroma.Config.get!([Local, :uploads])
uploader = Module.concat(Pleroma.Uploaders, target_uploader)
unless Code.ensure_loaded?(uploader) do
raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
end
target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
unless target_enabled? do
Pleroma.Config.put([Upload, :uploader], uploader)
end
Logger.info("Migrating files from local #{local_path} to #{to_string(uploader)}")
if delete? do
Logger.warn(
"Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
)
:timer.sleep(:timer.seconds(5))
end
uploads = File.ls!(local_path)
total_count = length(uploads)
uploads
|> Task.async_stream(
fn uuid ->
u_path = Path.join(local_path, uuid)
{name, path} =
cond do
File.dir?(u_path) ->
files = for file <- File.ls!(u_path), do: {{file, uuid}, Path.join([u_path, file])}
List.first(files)
File.exists?(u_path) ->
# {uuid, u_path}
raise "should_dedupe local storage not supported yet sorry"
end
{:ok, _} =
Upload.store({:from_local, name, path}, should_dedupe: false, uploader: uploader)
if delete? do
File.rm_rf!(u_path)
end
Logger.debug("uploaded: #{inspect(name)}")
end,
timeout: 150_000
)
|> Stream.chunk_every(@log_every)
|> Enum.reduce(0, fn done, count ->
count = count + length(done)
Logger.info("Uploaded #{count}/#{total_count} files")
count
end)
Logger.info("Done!")
end
def run(_) do
Logger.error("Usage: migrate_local_uploads UploaderName [--delete]")
end
end
......@@ -7,6 +7,11 @@ def name, do: @name
def version, do: @version
def named_version(), do: @name <> " " <> @version
def user_agent() do
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
end
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
@env Mix.env()
......
defmodule Pleroma.Plugs.UploadedMedia do
@moduledoc """
"""
import Plug.Conn
require Logger
@behaviour Plug
# no slashes
@path "media"
@cache_control %{
default: "public, max-age=1209600",
error: "public, must-revalidate, max-age=160"
}
def init(_opts) do
static_plug_opts =
[]
|> Keyword.put(:from, "__unconfigured_media_plug")
|> Keyword.put(:at, "/__unconfigured_media_plug")
|> Plug.Static.init()
%{static_plug_opts: static_plug_opts}
end
def call(conn = %{request_path: <<"/", @path, "/", file::binary>>}, opts) do
config = Pleroma.Config.get([Pleroma.Upload])
with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false),
{:ok, get_method} <- uploader.get_file(file) do
get_media(conn, get_method, proxy_remote, opts)
else
_ ->
conn
|> send_resp(500, "Failed")
|> halt()
end
end
def call(conn, _opts), do: conn
defp get_media(conn, {:static_dir, directory}, _, opts) do
static_opts =
Map.get(opts, :static_plug_opts)
|> Map.put(:at, [@path])
|> Map.put(:from, directory)
conn = Plug.Static.call(conn, static_opts)
if conn.halted do
conn
else
conn
|> send_resp(404, "Not found")
|> halt()
end
end
defp get_media(conn, {:url, url}, true, _) do
conn
|> Pleroma.ReverseProxy.call(url, Pleroma.Config.get([Pleroma.Upload, :proxy_opts], []))
end
defp get_media(conn, {:url, url}, _, _) do
conn
|> Phoenix.Controller.redirect(external: url)
|> halt()
end
defp get_media(conn, unknown, _, _) do
Logger.error("#{__MODULE__}: Unknown get startegy: #{inspect(unknown)}")
conn
|> send_resp(500, "Internal Error")
|> halt()
end
end
defmodule Pleroma.ReverseProxy do
@keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-none-match range)
@resp_cache_headers ~w(etag date last-modified cache-control)
@keep_resp_headers @resp_cache_headers ++
~w(content-type content-disposition content-length accept-ranges vary)
@default_cache_control_header "public, max-age=1209600"
@valid_resp_codes [200, 206, 304]
@max_read_duration :timer.minutes(2)
@max_body_length :infinity
@methods ~w(GET HEAD)
@moduledoc """
A reverse proxy.
Pleroma.ReverseProxy.call(conn, url, options)
It is not meant to be added into a plug pipeline, but to be called from another plug or controller.
Supports `#{inspect(@methods)}` HTTP methods, and only allows `#{inspect(@valid_resp_codes)}` status codes.
Responses are chunked to the client while downloading from the upstream.
Some request / responses headers are preserved:
* request: `#{inspect(@keep_req_headers)}`
* response: `#{inspect(@keep_resp_headers)}`
If no caching headers (`#{inspect(@resp_cache_headers)}`) are returned by upstream, `cache-control` will be
set to `#{inspect(@default_cache_control_header)}`.
Options:
* `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
errors. Any error during body processing will not be redirected as the response is chunked. This may expose
remote URL, clients IPs, ….
* `max_body_length` (default `#{inspect(@max_body_length)}`): limits the content length to be approximately the
specified length. It is validated with the `content-length` header and also verified when proxying.
* `max_read_duration` (default `#{inspect(@max_read_duration)}` ms): the total time the connection is allowed to
read from the remote upstream.
* `inline_content_types`:
* `true` will not alter `content-disposition` (up to the upstream),
* `false` will add `content-disposition: attachment` to any request,
* a list of whitelisted content types
* `keep_user_agent` will forward the client's user-agent to the upstream. This may be useful if the upstream is
doing content transformation (encoding, …) depending on the request.
* `req_headers`, `resp_headers` additional headers.
* `http`: options for [hackney](https://github.com/benoitc/hackney).
"""
@hackney Application.get_env(:pleroma, :hackney, :hackney)
@httpoison Application.get_env(:pleroma, :httpoison, HTTPoison)
@default_hackney_options [{:follow_redirect, true}]
@inline_content_types [
"image/gif",
"image/jpeg",
"image/jpg",
"image/png",
"image/svg+xml",
"audio/mpeg",
"audio/mp3",
"video/webm",
"video/mp4",
"video/quicktime"
]
require Logger
import Plug.Conn
@type option() ::
{:keep_user_agent, boolean}
| {:max_read_duration, :timer.time() | :infinity}
| {:max_body_length, non_neg_integer() | :infinity}
| {:http, []}
| {:req_headers, [{String.t(), String.t()}]}
| {:resp_headers, [{String.t(), String.t()}]}
| {:inline_content_types, boolean() | [String.t()]}
| {:redirect_on_failure, boolean()}
@spec call(Plug.Conn.t(), url :: String.t(), [option()]) :: Plug.Conn.t()
def call(conn = %{method: method}, url, opts \\ []) when method in @methods do
hackney_opts =
@default_hackney_options
|> Keyword.merge(Keyword.get(opts, :http, []))
|> @httpoison.process_request_options()
req_headers = build_req_headers(conn.req_headers, opts)
opts =
if filename = Pleroma.Web.MediaProxy.filename(url) do
Keyword.put_new(opts, :attachment_name, filename)
else
opts
end
with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
:ok <- header_lenght_constraint(headers, Keyword.get(opts, :max_body_length)) do
response(conn, client, url, code, headers, opts)
else
{:ok, code, headers} ->
head_response(conn, url, code, headers, opts)
|> halt()
{:error, {:invalid_http_response, code}} ->
Logger.error("#{__MODULE__}: request to #{inspect(url)} failed with HTTP status #{code}")
conn
|> error_or_redirect(
url,
code,
"Request failed: " <> Plug.Conn.Status.reason_phrase(code),
opts
)
|> halt()
{:error, error} ->
Logger.error("#{__MODULE__}: request to #{inspect(url)} failed: #{inspect(error)}")
conn
|> error_or_redirect(url, 500, "Request failed", opts)
|> halt()
end
end
def call(conn, _, _) do
conn
|> send_resp(400, Plug.Conn.Status.reason_phrase(400))
|> halt()
end
defp request(method, url, headers, hackney_opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
case @hackney.request(method, url, headers, "", hackney_opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
{:ok, code, headers} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers)}
{:ok, code, _, _} ->
{:error, {:invalid_http_response, code}}
{:error, error} ->
{:error, error}
end
end
defp response(conn, client, url, status, headers, opts) do
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> send_chunked(status)
|> chunk_reply(client, opts)
case result do
{:ok, conn} ->
halt(conn)
{:error, :closed, conn} ->
:hackney.close(client)
halt(conn)
{:error, error, conn} ->
Logger.warn(
"#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}"
)
:hackney.close(client)
halt(conn)
end
end
defp chunk_reply(conn, client, opts) do
chunk_reply(conn, client, opts, 0, 0)
end
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
with {:ok, duration} <-
check_read_duration(
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
),
{:ok, data} <- @hackney.stream_body(client),
{:ok, duration} <- increase_read_duration(duration),
sent_so_far = sent_so_far + byte_size(data),
:ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)),
{:ok, conn} <- chunk(conn, data) do
chunk_reply(conn, client, opts, sent_so_far, duration)
else
:done -> {:ok, conn}
{:error, error} -> {:error, error, conn}
end
end
defp head_response(conn, _url, code, headers, opts) do
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> send_resp(code, "")
end
defp error_or_redirect(conn, url, code, body, opts) do
if Keyword.get(opts, :redirect_on_failure, false) do
conn
|> Phoenix.Controller.redirect(external: url)
|> halt()
else
conn
|> send_resp(code, body)
|> halt
end
end
defp downcase_headers(headers) do
Enum.map(headers, fn {k, v} ->
{String.downcase(k), v}
end)
end
defp put_resp_headers(conn, headers) do
Enum.reduce(headers, conn, fn {k, v}, conn ->
put_resp_header(conn, k, v)
end)
end
defp build_req_headers(headers, opts) do
headers =
headers
|> downcase_headers()
|> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
|> (fn headers ->
headers = headers ++ Keyword.get(opts, :req_headers, [])
if Keyword.get(opts, :keep_user_agent, false) do
List.keystore(
headers,
"user-agent",
0,
{"user-agent", Pleroma.Application.user_agent()}
)
else
headers
end
end).()
end
defp build_resp_headers(headers, opts) do
headers =
headers
|> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
|> build_resp_cache_headers(opts)
|> build_resp_content_disposition_header(opts)
|> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).()
end
defp build_resp_cache_headers(headers, opts) do
has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
if has_cache? do
headers
else
List.keystore(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
end
end
defp build_resp_content_disposition_header(headers, opts) do
opt = Keyword.get(opts, :inline_content_types, @inline_content_types)
{_, content_type} =
List.keyfind(headers, "content-type", 0, {"content-type", "application/octect-stream"})
attachment? =
cond do
is_list(opt) && !Enum.member?(opt, content_type) -> true
opt == false -> true
true -> false
end
if attachment? do
disposition = "attachment; filename=" <> Keyword.get(opts, :attachment_name, "attachment")
List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
else
headers
end
end
defp header_lenght_constraint(headers, limit) when is_integer(limit) and limit > 0 do
with {_, size} <- List.keyfind(headers, "content-length", 0),
{size, _} <- Integer.parse(size),
true <- size <= limit do
:ok
else
false ->
{:error, :body_too_large}
_ ->
:ok
end
end
defp header_lenght_constraint(_, _), do: :ok
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
{:error, :body_too_large}
end
defp body_size_constraint(_, _), do: :ok
defp check_read_duration(duration, max)
when is_integer(duration) and is_integer(max) and max > 0 do
if duration > max do
{:error, :read_duration_exceeded}
else
Logger.debug("Duration #{inspect(duration)}")
{:ok, {duration, :erlang.system_time(:millisecond)}}
end
end
defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
defp increase_read_duration({previous_duration, started})
when is_integer(previous_duration) and is_integer(started) do
duration = :erlang.system_time(:millisecond) - started
{:ok, previous_duration + duration}
end
defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit}
end
end
defmodule Pleroma.Upload do
alias Ecto.UUID
require Logger
@type upload_option ::
{:dedupe, boolean()} | {:size_limit, non_neg_integer()} | {:uploader, module()}
@type upload_source ::
Plug.Upload.t() | data_uri_string() ::
String.t() | {:from_local, name :: String.t(), uuid :: String.t(), path :: String.t()}
@spec store(upload_source, options :: [upload_option()]) :: {:ok, Map.t()} | {:error, any()}
def store(upload, opts \\ []) do
opts = get_opts(opts)
with {:ok, name, uuid, path, content_type} <- process_upload(upload, opts),
_ <- strip_exif_data(content_type, path),
{:ok, url_spec} <- opts.uploader.put_file(name, uuid, path, content_type, opts) do
{:ok,
%{
"type" => "Image",
"url" => [
%{
"type" => "Link",
"mediaType" => content_type,
"href" => url_from_spec(url_spec)
}
],
"name" => name
}}
else