Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Pleroma
pleroma
Commits
b19597f6
Verified
Commit
b19597f6
authored
Nov 23, 2018
by
href
Browse files
reverse proxy / uploads
parent
52ce3685
Changes
23
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
b19597f6
...
...
@@ -6,6 +6,9 @@
/uploads
/test/uploads
/.elixir_ls
/test/fixtures/test_tmp.txt
/test/fixtures/image_tmp.jpg
/doc
# Prevent committing custom emojis
/priv/static/emoji/custom/*
...
...
@@ -28,4 +31,4 @@ erl_crash.dump
.env
# Editor config
/.vscode
\ No newline at end of file
/.vscode
config/config.exs
View file @
b19597f6
...
...
@@ -12,16 +12,15 @@
config
:pleroma
,
Pleroma
.
Upload
,
uploader:
Pleroma
.
Uploaders
.
Local
,
strip_exif:
false
strip_exif:
false
,
proxy_remote:
false
,
proxy_opts:
[
inline_content_types:
true
,
keep_user_agent:
true
]
config
:pleroma
,
Pleroma
.
Uploaders
.
Local
,
uploads:
"uploads"
,
uploads_url:
"{{base_url}}/media/{{file}}"
config
:pleroma
,
Pleroma
.
Uploaders
.
Local
,
uploads:
"uploads"
config
:pleroma
,
Pleroma
.
Uploaders
.
S3
,
bucket:
nil
,
public_endpoint:
"https://s3.amazonaws.com"
,
force_media_proxy:
false
public_endpoint:
"https://s3.amazonaws.com"
config
:pleroma
,
Pleroma
.
Uploaders
.
MDII
,
cgi:
"https://mdii.sakura.ne.jp/mdii-post.cgi"
,
...
...
@@ -150,9 +149,11 @@
config
:pleroma
,
:media_proxy
,
enabled:
false
,
redirect_on_failure:
true
# base_url: "https://cache.pleroma.social"
# base_url: "https://cache.pleroma.social",
proxy_opts:
[
# inline_content_types: [] | false | true,
# http: [:insecure]
]
config
:pleroma
,
:chat
,
enabled:
true
...
...
config/test.exs
View file @
b19597f6
...
...
@@ -9,7 +9,7 @@
# Print only warnings and errors during test
config
:logger
,
level:
:warn
config
:pleroma
,
Pleroma
.
Upload
,
uploads:
"test/uploads"
config
:pleroma
,
Pleroma
.
Upload
ers
.
Local
,
uploads:
"test/uploads"
# Configure your database
config
:pleroma
,
Pleroma
.
Repo
,
...
...
lib/mix/tasks/migrate_local_uploads.ex
0 → 100644
View file @
b19597f6
defmodule
Mix
.
Tasks
.
MigrateLocalUploads
do
use
Mix
.
Task
import
Mix
.
Ecto
alias
Pleroma
.
{
Upload
,
Uploaders
.
Local
,
Uploaders
.
S3
}
require
Logger
@log_every
50
@shortdoc
"Migrate uploads from local to remote storage"
def
run
([
target_uploader
|
args
])
do
delete?
=
Enum
.
member?
(
args
,
"--delete"
)
Application
.
ensure_all_started
(
:pleroma
)
local_path
=
Pleroma
.
Config
.
get!
([
Local
,
:uploads
])
uploader
=
Module
.
concat
(
Pleroma
.
Uploaders
,
target_uploader
)
unless
Code
.
ensure_loaded?
(
uploader
)
do
raise
(
"The uploader
#{
inspect
(
uploader
)
}
is not an existing/loaded module."
)
end
target_enabled?
=
Pleroma
.
Config
.
get
([
Upload
,
:uploader
])
==
uploader
unless
target_enabled?
do
Pleroma
.
Config
.
put
([
Upload
,
:uploader
],
uploader
)
end
Logger
.
info
(
"Migrating files from local
#{
local_path
}
to
#{
to_string
(
uploader
)
}
"
)
if
delete?
do
Logger
.
warn
(
"Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
)
:timer
.
sleep
(
:timer
.
seconds
(
5
))
end
uploads
=
File
.
ls!
(
local_path
)
total_count
=
length
(
uploads
)
uploads
|>
Task
.
async_stream
(
fn
uuid
->
u_path
=
Path
.
join
(
local_path
,
uuid
)
{
name
,
path
}
=
cond
do
File
.
dir?
(
u_path
)
->
files
=
for
file
<-
File
.
ls!
(
u_path
),
do
:
{{
file
,
uuid
},
Path
.
join
([
u_path
,
file
])}
List
.
first
(
files
)
File
.
exists?
(
u_path
)
->
# {uuid, u_path}
raise
"should_dedupe local storage not supported yet sorry"
end
{
:ok
,
_
}
=
Upload
.
store
({
:from_local
,
name
,
path
},
should_dedupe:
false
,
uploader:
uploader
)
if
delete?
do
File
.
rm_rf!
(
u_path
)
end
Logger
.
debug
(
"uploaded:
#{
inspect
(
name
)
}
"
)
end
,
timeout:
150_000
)
|>
Stream
.
chunk_every
(
@log_every
)
|>
Enum
.
reduce
(
0
,
fn
done
,
count
->
count
=
count
+
length
(
done
)
Logger
.
info
(
"Uploaded
#{
count
}
/
#{
total_count
}
files"
)
count
end
)
Logger
.
info
(
"Done!"
)
end
def
run
(
_
)
do
Logger
.
error
(
"Usage: migrate_local_uploads UploaderName [--delete]"
)
end
end
lib/pleroma/application.ex
View file @
b19597f6
...
...
@@ -7,6 +7,11 @@ def name, do: @name
def
version
,
do
:
@version
def
named_version
(),
do
:
@name
<>
" "
<>
@version
def
user_agent
()
do
info
=
"
#{
Pleroma
.
Web
.
base_url
()
}
<
#{
Pleroma
.
Config
.
get
([
:instance
,
:email
],
""
)
}
>"
named_version
()
<>
"; "
<>
info
end
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
@env
Mix
.
env
()
...
...
lib/pleroma/plugs/uploaded_media.ex
0 → 100644
View file @
b19597f6
defmodule
Pleroma
.
Plugs
.
UploadedMedia
do
@moduledoc
"""
"""
import
Plug
.
Conn
require
Logger
@behaviour
Plug
# no slashes
@path
"media"
@cache_control
%{
default:
"public, max-age=1209600"
,
error:
"public, must-revalidate, max-age=160"
}
def
init
(
_opts
)
do
static_plug_opts
=
[]
|>
Keyword
.
put
(
:from
,
"__unconfigured_media_plug"
)
|>
Keyword
.
put
(
:at
,
"/__unconfigured_media_plug"
)
|>
Plug
.
Static
.
init
()
%{
static_plug_opts:
static_plug_opts
}
end
def
call
(
conn
=
%{
request_path:
<<
"/"
,
@path
,
"/"
,
file
::
binary
>>
},
opts
)
do
config
=
Pleroma
.
Config
.
get
([
Pleroma
.
Upload
])
with
uploader
<-
Keyword
.
fetch!
(
config
,
:uploader
),
proxy_remote
=
Keyword
.
get
(
config
,
:proxy_remote
,
false
),
{
:ok
,
get_method
}
<-
uploader
.
get_file
(
file
)
do
get_media
(
conn
,
get_method
,
proxy_remote
,
opts
)
else
_
->
conn
|>
send_resp
(
500
,
"Failed"
)
|>
halt
()
end
end
def
call
(
conn
,
_opts
),
do
:
conn
defp
get_media
(
conn
,
{
:static_dir
,
directory
},
_
,
opts
)
do
static_opts
=
Map
.
get
(
opts
,
:static_plug_opts
)
|>
Map
.
put
(
:at
,
[
@path
])
|>
Map
.
put
(
:from
,
directory
)
conn
=
Plug
.
Static
.
call
(
conn
,
static_opts
)
if
conn
.
halted
do
conn
else
conn
|>
send_resp
(
404
,
"Not found"
)
|>
halt
()
end
end
defp
get_media
(
conn
,
{
:url
,
url
},
true
,
_
)
do
conn
|>
Pleroma
.
ReverseProxy
.
call
(
url
,
Pleroma
.
Config
.
get
([
Pleroma
.
Upload
,
:proxy_opts
],
[]))
end
defp
get_media
(
conn
,
{
:url
,
url
},
_
,
_
)
do
conn
|>
Phoenix
.
Controller
.
redirect
(
external:
url
)
|>
halt
()
end
defp
get_media
(
conn
,
unknown
,
_
,
_
)
do
Logger
.
error
(
"
#{
__MODULE__
}
: Unknown get startegy:
#{
inspect
(
unknown
)
}
"
)
conn
|>
send_resp
(
500
,
"Internal Error"
)
|>
halt
()
end
end
lib/pleroma/reverse_proxy.ex
0 → 100644
View file @
b19597f6
defmodule
Pleroma
.
ReverseProxy
do
@keep_req_headers
~w(accept user-agent accept-encoding cache-control if-modified-since if-none-match range)
@resp_cache_headers
~w(etag date last-modified cache-control)
@keep_resp_headers
@resp_cache_headers
++
~w(content-type content-disposition content-length accept-ranges vary)
@default_cache_control_header
"public, max-age=1209600"
@valid_resp_codes
[
200
,
206
,
304
]
@max_read_duration
:timer
.
minutes
(
2
)
@max_body_length
:infinity
@methods
~w(GET HEAD)
@moduledoc
"""
A reverse proxy.
Pleroma.ReverseProxy.call(conn, url, options)
It is not meant to be added into a plug pipeline, but to be called from another plug or controller.
Supports `#{inspect(@methods)}` HTTP methods, and only allows `#{inspect(@valid_resp_codes)}` status codes.
Responses are chunked to the client while downloading from the upstream.
Some request / responses headers are preserved:
* request: `#{inspect(@keep_req_headers)}`
* response: `#{inspect(@keep_resp_headers)}`
If no caching headers (`#{inspect(@resp_cache_headers)}`) are returned by upstream, `cache-control` will be
set to `#{inspect(@default_cache_control_header)}`.
Options:
* `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
errors. Any error during body processing will not be redirected as the response is chunked. This may expose
remote URL, clients IPs, ….
* `max_body_length` (default `#{inspect(@max_body_length)}`): limits the content length to be approximately the
specified length. It is validated with the `content-length` header and also verified when proxying.
* `max_read_duration` (default `#{inspect(@max_read_duration)}` ms): the total time the connection is allowed to
read from the remote upstream.
* `inline_content_types`:
* `true` will not alter `content-disposition` (up to the upstream),
* `false` will add `content-disposition: attachment` to any request,
* a list of whitelisted content types
* `keep_user_agent` will forward the client's user-agent to the upstream. This may be useful if the upstream is
doing content transformation (encoding, …) depending on the request.
* `req_headers`, `resp_headers` additional headers.
* `http`: options for [hackney](https://github.com/benoitc/hackney).
"""
@hackney
Application
.
get_env
(
:pleroma
,
:hackney
,
:hackney
)
@httpoison
Application
.
get_env
(
:pleroma
,
:httpoison
,
HTTPoison
)
@default_hackney_options
[{
:follow_redirect
,
true
}]
@inline_content_types
[
"image/gif"
,
"image/jpeg"
,
"image/jpg"
,
"image/png"
,
"image/svg+xml"
,
"audio/mpeg"
,
"audio/mp3"
,
"video/webm"
,
"video/mp4"
,
"video/quicktime"
]
require
Logger
import
Plug
.
Conn
@type
option
()
::
{
:keep_user_agent
,
boolean
}
|
{
:max_read_duration
,
:timer
.
time
()
|
:infinity
}
|
{
:max_body_length
,
non_neg_integer
()
|
:infinity
}
|
{
:http
,
[]}
|
{
:req_headers
,
[{
String
.
t
(),
String
.
t
()}]}
|
{
:resp_headers
,
[{
String
.
t
(),
String
.
t
()}]}
|
{
:inline_content_types
,
boolean
()
|
[
String
.
t
()]}
|
{
:redirect_on_failure
,
boolean
()}
@spec
call
(
Plug
.
Conn
.
t
(),
url
::
String
.
t
(),
[
option
()])
::
Plug
.
Conn
.
t
()
def
call
(
conn
=
%{
method:
method
},
url
,
opts
\\
[])
when
method
in
@methods
do
hackney_opts
=
@default_hackney_options
|>
Keyword
.
merge
(
Keyword
.
get
(
opts
,
:http
,
[]))
|>
@httpoison
.
process_request_options
()
req_headers
=
build_req_headers
(
conn
.
req_headers
,
opts
)
opts
=
if
filename
=
Pleroma
.
Web
.
MediaProxy
.
filename
(
url
)
do
Keyword
.
put_new
(
opts
,
:attachment_name
,
filename
)
else
opts
end
with
{
:ok
,
code
,
headers
,
client
}
<-
request
(
method
,
url
,
req_headers
,
hackney_opts
),
:ok
<-
header_lenght_constraint
(
headers
,
Keyword
.
get
(
opts
,
:max_body_length
))
do
response
(
conn
,
client
,
url
,
code
,
headers
,
opts
)
else
{
:ok
,
code
,
headers
}
->
head_response
(
conn
,
url
,
code
,
headers
,
opts
)
|>
halt
()
{
:error
,
{
:invalid_http_response
,
code
}}
->
Logger
.
error
(
"
#{
__MODULE__
}
: request to
#{
inspect
(
url
)
}
failed with HTTP status
#{
code
}
"
)
conn
|>
error_or_redirect
(
url
,
code
,
"Request failed: "
<>
Plug
.
Conn
.
Status
.
reason_phrase
(
code
),
opts
)
|>
halt
()
{
:error
,
error
}
->
Logger
.
error
(
"
#{
__MODULE__
}
: request to
#{
inspect
(
url
)
}
failed:
#{
inspect
(
error
)
}
"
)
conn
|>
error_or_redirect
(
url
,
500
,
"Request failed"
,
opts
)
|>
halt
()
end
end
def
call
(
conn
,
_
,
_
)
do
conn
|>
send_resp
(
400
,
Plug
.
Conn
.
Status
.
reason_phrase
(
400
))
|>
halt
()
end
defp
request
(
method
,
url
,
headers
,
hackney_opts
)
do
Logger
.
debug
(
"
#{
__MODULE__
}
#{
method
}
#{
url
}
#{
inspect
(
headers
)
}
"
)
method
=
method
|>
String
.
downcase
()
|>
String
.
to_existing_atom
()
case
@hackney
.
request
(
method
,
url
,
headers
,
""
,
hackney_opts
)
do
{
:ok
,
code
,
headers
,
client
}
when
code
in
@valid_resp_codes
->
{
:ok
,
code
,
downcase_headers
(
headers
),
client
}
{
:ok
,
code
,
headers
}
when
code
in
@valid_resp_codes
->
{
:ok
,
code
,
downcase_headers
(
headers
)}
{
:ok
,
code
,
_
,
_
}
->
{
:error
,
{
:invalid_http_response
,
code
}}
{
:error
,
error
}
->
{
:error
,
error
}
end
end
defp
response
(
conn
,
client
,
url
,
status
,
headers
,
opts
)
do
result
=
conn
|>
put_resp_headers
(
build_resp_headers
(
headers
,
opts
))
|>
send_chunked
(
status
)
|>
chunk_reply
(
client
,
opts
)
case
result
do
{
:ok
,
conn
}
->
halt
(
conn
)
{
:error
,
:closed
,
conn
}
->
:hackney
.
close
(
client
)
halt
(
conn
)
{
:error
,
error
,
conn
}
->
Logger
.
warn
(
"
#{
__MODULE__
}
request to
#{
url
}
failed while reading/chunking:
#{
inspect
(
error
)
}
"
)
:hackney
.
close
(
client
)
halt
(
conn
)
end
end
defp
chunk_reply
(
conn
,
client
,
opts
)
do
chunk_reply
(
conn
,
client
,
opts
,
0
,
0
)
end
defp
chunk_reply
(
conn
,
client
,
opts
,
sent_so_far
,
duration
)
do
with
{
:ok
,
duration
}
<-
check_read_duration
(
duration
,
Keyword
.
get
(
opts
,
:max_read_duration
,
@max_read_duration
)
),
{
:ok
,
data
}
<-
@hackney
.
stream_body
(
client
),
{
:ok
,
duration
}
<-
increase_read_duration
(
duration
),
sent_so_far
=
sent_so_far
+
byte_size
(
data
),
:ok
<-
body_size_constraint
(
sent_so_far
,
Keyword
.
get
(
opts
,
:max_body_size
)),
{
:ok
,
conn
}
<-
chunk
(
conn
,
data
)
do
chunk_reply
(
conn
,
client
,
opts
,
sent_so_far
,
duration
)
else
:done
->
{
:ok
,
conn
}
{
:error
,
error
}
->
{
:error
,
error
,
conn
}
end
end
defp
head_response
(
conn
,
_url
,
code
,
headers
,
opts
)
do
conn
|>
put_resp_headers
(
build_resp_headers
(
headers
,
opts
))
|>
send_resp
(
code
,
""
)
end
defp
error_or_redirect
(
conn
,
url
,
code
,
body
,
opts
)
do
if
Keyword
.
get
(
opts
,
:redirect_on_failure
,
false
)
do
conn
|>
Phoenix
.
Controller
.
redirect
(
external:
url
)
|>
halt
()
else
conn
|>
send_resp
(
code
,
body
)
|>
halt
end
end
defp
downcase_headers
(
headers
)
do
Enum
.
map
(
headers
,
fn
{
k
,
v
}
->
{
String
.
downcase
(
k
),
v
}
end
)
end
defp
put_resp_headers
(
conn
,
headers
)
do
Enum
.
reduce
(
headers
,
conn
,
fn
{
k
,
v
},
conn
->
put_resp_header
(
conn
,
k
,
v
)
end
)
end
defp
build_req_headers
(
headers
,
opts
)
do
headers
=
headers
|>
downcase_headers
()
|>
Enum
.
filter
(
fn
{
k
,
_
}
->
k
in
@keep_req_headers
end
)
|>
(
fn
headers
->
headers
=
headers
++
Keyword
.
get
(
opts
,
:req_headers
,
[])
if
Keyword
.
get
(
opts
,
:keep_user_agent
,
false
)
do
List
.
keystore
(
headers
,
"user-agent"
,
0
,
{
"user-agent"
,
Pleroma
.
Application
.
user_agent
()}
)
else
headers
end
end
)
.
()
end
defp
build_resp_headers
(
headers
,
opts
)
do
headers
=
headers
|>
Enum
.
filter
(
fn
{
k
,
_
}
->
k
in
@keep_resp_headers
end
)
|>
build_resp_cache_headers
(
opts
)
|>
build_resp_content_disposition_header
(
opts
)
|>
(
fn
headers
->
headers
++
Keyword
.
get
(
opts
,
:resp_headers
,
[])
end
)
.
()
end
defp
build_resp_cache_headers
(
headers
,
opts
)
do
has_cache?
=
Enum
.
any?
(
headers
,
fn
{
k
,
_
}
->
k
in
@resp_cache_headers
end
)
if
has_cache?
do
headers
else
List
.
keystore
(
headers
,
"cache-control"
,
0
,
{
"cache-control"
,
@default_cache_control_header
})
end
end
defp
build_resp_content_disposition_header
(
headers
,
opts
)
do
opt
=
Keyword
.
get
(
opts
,
:inline_content_types
,
@inline_content_types
)
{
_
,
content_type
}
=
List
.
keyfind
(
headers
,
"content-type"
,
0
,
{
"content-type"
,
"application/octect-stream"
})
attachment?
=
cond
do
is_list
(
opt
)
&&
!Enum
.
member?
(
opt
,
content_type
)
->
true
opt
==
false
->
true
true
->
false
end
if
attachment?
do
disposition
=
"attachment; filename="
<>
Keyword
.
get
(
opts
,
:attachment_name
,
"attachment"
)
List
.
keystore
(
headers
,
"content-disposition"
,
0
,
{
"content-disposition"
,
disposition
})
else
headers
end
end
defp
header_lenght_constraint
(
headers
,
limit
)
when
is_integer
(
limit
)
and
limit
>
0
do
with
{
_
,
size
}
<-
List
.
keyfind
(
headers
,
"content-length"
,
0
),
{
size
,
_
}
<-
Integer
.
parse
(
size
),
true
<-
size
<=
limit
do
:ok
else
false
->
{
:error
,
:body_too_large
}
_
->
:ok
end
end
defp
header_lenght_constraint
(
_
,
_
),
do
:
:ok
defp
body_size_constraint
(
size
,
limit
)
when
is_integer
(
limit
)
and
limit
>
0
and
size
>=
limit
do
{
:error
,
:body_too_large
}
end
defp
body_size_constraint
(
_
,
_
),
do
:
:ok
defp
check_read_duration
(
duration
,
max
)
when
is_integer
(
duration
)
and
is_integer
(
max
)
and
max
>
0
do
if
duration
>
max
do
{
:error
,
:read_duration_exceeded
}
else
Logger
.
debug
(
"Duration
#{
inspect
(
duration
)
}
"
)
{
:ok
,
{
duration
,
:erlang
.
system_time
(
:millisecond
)}}
end
end
defp
check_read_duration
(
_
,
_
),
do
:
{
:ok
,
:no_duration_limit
,
:no_duration_limit
}
defp
increase_read_duration
({
previous_duration
,
started
})
when
is_integer
(
previous_duration
)
and
is_integer
(
started
)
do
duration
=
:erlang
.
system_time
(
:millisecond
)
-
started
{
:ok
,
previous_duration
+
duration
}
end
defp
increase_read_duration
(
_
)
do
{
:ok
,
:no_duration_limit
,
:no_duration_limit
}
end
end
lib/pleroma/upload.ex
View file @
b19597f6
defmodule
Pleroma
.
Upload
do
alias
Ecto
.
UUID
require
Logger
@type
upload_option
::
{
:dedupe
,
boolean
()}
|
{
:size_limit
,
non_neg_integer
()}
|
{
:uploader
,
module
()}
@type
upload_source
::
Plug
.
Upload
.
t
()
|
data_uri_string
()
::
String
.
t
()
|
{
:from_local
,
name
::
String
.
t
(),
uuid
::
String
.
t
(),
path
::
String
.
t
()}
@spec
store
(
upload_source
,
options
::
[
upload_option
()])
::
{
:ok
,
Map
.
t
()}
|
{
:error
,
any
()}
def
store
(
upload
,
opts
\\
[])
do
opts
=
get_opts
(
opts
)
with
{
:ok
,
name
,
uuid
,
path
,
content_type
}
<-
process_upload
(
upload
,
opts
),
_
<-
strip_exif_data
(
content_type
,
path
),
{
:ok
,
url_spec
}
<-
opts
.
uploader
.
put_file
(
name
,
uuid
,
path
,
content_type
,
opts
)
do
{
:ok
,
%{
"type"
=>
"Image"
,
"url"
=>
[
%{
"type"
=>
"Link"
,
"mediaType"
=>
content_type
,
"href"
=>
url_from_spec
(
url_spec
)
}
],
"name"
=>
name
}}
else