Commit fbd71abc authored by Your New SJW Waifu's avatar Your New SJW Waifu 💬

Merge remote-tracking branch 'upstream/develop' into neckbeard

parents 3e4ce4b7 023f726d
Pipeline #32021 failed with stages
in 44 minutes and 59 seconds
......@@ -27,6 +27,8 @@ erl_crash.dump
# variables.
/config/*.secret.exs
/config/generated_config.exs
/config/*.env
# Database setup file, some may forget to delete it
/config/setup_db.psql
......
......@@ -25,6 +25,8 @@ before_script:
- apt-get update && apt-get install -y cmake
- mix local.hex --force
- mix local.rebar --force
- apt-get -qq update
- apt-get install -y libmagic-dev
build:
stage: build
......
......@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed
- **Breaking** Requires `libmagic` (or `file`) to guess file types.
- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
- Search: Users are now findable by their urls.
......
......@@ -802,6 +802,8 @@
timeout: 300_000
]
config :pleroma, :majic_pool, size: 2
private_instance? = :if_instance_is_private
config :pleroma, :restrict_unauthenticated,
......
......@@ -3708,5 +3708,19 @@
]
}
]
},
%{
group: :pleroma,
key: :majic_pool,
type: :group,
description: "Majic/libmagic configuration",
children: [
%{
key: :size,
type: :integer,
description: "Number of majic workers to start.",
suggestions: [2]
}
]
}
]
......@@ -40,3 +40,5 @@ If any of the options are left unspecified, you will be prompted interactively.
- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
- `--skip-release-env` - skip generation the release environment file
- `--release-env-file` - release environment file path
# Generate release environment file
```sh tab="OTP"
./bin/pleroma_ctl release_env gen
```
```sh tab="From Source"
mix pleroma.release_env gen
```
......@@ -71,7 +71,7 @@ Feel free to contact us to be added to this list!
### Indigenous
- Homepage: <https://indigenous.realize.be/>
- Source Code: <https://github.com/swentel/indigenous-android/>
- Contact: [@realize.be@realize.be](@realize.be@realize.be)
- Contact: [@swentel@realize.be](https://realize.be)
- Platforms: Android
- Features: No Streaming
......
......@@ -13,6 +13,7 @@ It assumes that you have administrative rights, either as root or a user with [s
* `erlang-parsetools`
* `erlang-xmerl`
* `git`
* `file-dev`
* Development Tools
* `cmake`
......@@ -42,7 +43,7 @@ sudo apk upgrade
* Install some tools, which are needed later:
```shell
sudo apk add git build-base cmake
sudo apk add git build-base cmake file-dev
```
### Install Elixir and Erlang
......
......@@ -10,6 +10,7 @@ This guide will assume that you have administrative rights, either as root or a
* `git`
* `base-devel`
* `cmake`
* `file`
#### Optional packages used in this guide
......@@ -30,7 +31,7 @@ sudo pacman -Syu
* Install some of the above mentioned programs:
```shell
sudo pacman -S git base-devel elixir cmake
sudo pacman -S git base-devel elixir cmake file
```
### Install PostgreSQL
......
......@@ -10,6 +10,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
* `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
* `erlang-dev`
* `erlang-nox`
* `libmagic-dev`
* `git`
* `build-essential`
* `cmake`
......@@ -34,7 +35,7 @@ sudo apt full-upgrade
* Install some of the above mentioned programs:
```shell
sudo apt install git build-essential postgresql postgresql-contrib cmake
sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
```
### Install Elixir and Erlang
......@@ -100,6 +101,7 @@ sudo -Hu pleroma mix deps.get
mv config/{generated_config.exs,prod.secret.exs}
```
* The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
```shell
......@@ -180,6 +182,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
```
* Edit the service file and make sure that all paths fit your installation
* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
* Enable and start `pleroma.service`:
```shell
......
......@@ -17,6 +17,7 @@
- `git`
- `build-essential`
- `cmake`
- `libmagic-dev`
#### このガイドで利用している追加パッケージ
......@@ -36,7 +37,7 @@ sudo apt full-upgrade
* 上記に挙げたパッケージをインストールしておきます。
```
sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick
sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick libmagic-dev
```
### ElixirとErlangをインストールします
......
......@@ -29,6 +29,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
* `dev-lang/elixir`
* `dev-vcs/git`
* `dev-util/cmake`
* `sys-apps/file`
#### Optional ebuilds used in this guide
......@@ -50,7 +51,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
* Emerge all required the required and suggested software in one go:
```shell
# emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
# emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file
```
If you would not like to install the optional packages, remove them from this line.
......
......@@ -27,17 +27,18 @@ Other than things bundled in the OTP release Pleroma depends on:
* PostgreSQL (also utilizes extensions in postgresql-contrib)
* nginx (could be swapped with another reverse proxy but this guide covers only it)
* certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
* libmagic/file
=== "Alpine"
```
echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
apk update
apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev
```
=== "Debian/Ubuntu"
```
apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev
```
### Installing optional packages
......@@ -148,6 +149,9 @@ chown -R pleroma /etc/pleroma
# Run the config generator
su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
# Run the environment file generator.
su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
# Create the postgres database
su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
......@@ -158,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
# su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
# Start the instance to verify that everything is working as expected
su pleroma -s $SHELL -lc "./bin/pleroma daemon"
su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
sleep 20 && curl http://localhost:4000/api/v1/instance
......@@ -310,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre
## Questions
Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
......@@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid"
directory=/opt/pleroma
healthcheck_delay=60
healthcheck_timer=30
export $(cat /opt/pleroma/config/pleroma.env)
: ${pleroma_port:-4000}
......
......@@ -17,6 +17,8 @@ Environment="MIX_ENV=prod"
Environment="HOME=/var/lib/pleroma"
; Path to the folder containing the Pleroma installation.
WorkingDirectory=/opt/pleroma
; Path to the environment file. the file contains RELEASE_COOKIE and etc
EnvironmentFile=/opt/pleroma/config/pleroma.env
; Path to the Mix binary.
ExecStart=/usr/bin/mix phx.server
......
......@@ -36,7 +36,9 @@ def run(["gen" | rest]) do
listen_port: :string,
strip_uploads: :string,
anonymize_uploads: :string,
dedupe_uploads: :string
dedupe_uploads: :string,
skip_release_env: :boolean,
release_env_file: :string
],
aliases: [
o: :output,
......@@ -241,6 +243,24 @@ def run(["gen" | rest]) do
write_robots_txt(static_dir, indexable, template_dir)
if Keyword.get(options, :skip_release_env, false) do
shell_info("""
Release environment file is skip. Please generate the release env file before start.
`MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
""")
else
shell_info("Generation the environment file:")
release_env_args =
with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
["gen", "--path", path]
else
_ -> ["gen"]
end
Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
end
shell_info(
"\n All files successfully written! Refer to the installation instructions for your platform for next steps."
)
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.ReleaseEnv do
use Mix.Task
import Mix.Pleroma
@shortdoc "Generate Pleroma environment file."
@moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
def run(["gen" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [
force: :boolean,
path: :string
],
aliases: [
p: :path,
f: :force
]
)
file_path =
get_option(
options,
:path,
"Environment file path",
"./config/pleroma.env"
)
env_path = Path.expand(file_path)
proceed? =
if File.exists?(env_path) do
get_option(
options,
:force,
"Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
"n"
) === "y"
else
true
end
if proceed? do
case do_generate(env_path) do
{:error, reason} ->
shell_error(
File.Error.message(%{action: "write to file", reason: reason, path: env_path})
)
_ ->
shell_info("\nThe file generated: #{env_path}.\n")
shell_info("""
WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
Example:
chmod 0444 #{file_path}
chattr +i #{file_path}
""")
end
else
shell_info("\nThe file is exist. #{env_path}.\n")
end
end
def do_generate(path) do
content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
File.mkdir_p!(Path.dirname(path))
File.write(path, content)
end
end
......@@ -419,7 +419,7 @@ def run(["list"]) do
|> Enum.each(fn user ->
shell_info(
"#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
user.locked
user.is_locked
}, deactivated: #{user.deactivated}"
)
end)
......@@ -447,10 +447,10 @@ defp set_admin(user, value) do
defp set_locked(user, value) do
{:ok, user} =
user
|> Changeset.change(%{locked: value})
|> Changeset.change(%{is_locked: value})
|> User.update_and_set_cache()
shell_info("Locked status of #{user.nickname}: #{user.locked}")
shell_info("Locked status of #{user.nickname}: #{user.is_locked}")
user
end
......
......@@ -95,6 +95,7 @@ def start(_type, _args) do
[
Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
{Oban, Config.get(Oban)}
] ++
task_children(@env) ++
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MIME do
@moduledoc """
Returns the mime-type of a binary and optionally a normalized file-name.
"""
@default "application/octet-stream"
@read_bytes 35
@spec file_mime_type(String.t(), String.t()) ::
{:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
def file_mime_type(path, filename) do
with {:ok, content_type} <- file_mime_type(path),
filename <- fix_extension(filename, content_type) do
{:ok, content_type, filename}
end
end
@spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
def file_mime_type(filename) do
File.open(filename, [:read], fn f ->
check_mime_type(IO.binread(f, @read_bytes))
end)
end
def bin_mime_type(binary, filename) do
with {:ok, content_type} <- bin_mime_type(binary),
filename <- fix_extension(filename, content_type) do
{:ok, content_type, filename}
end
end
@spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
{:ok, check_mime_type(head)}
end
def bin_mime_type(_), do: :error
def mime_type(<<_::binary>>), do: {:ok, @default}
defp fix_extension(filename, content_type) do
parts = String.split(filename, ".")
new_filename =
if length(parts) > 1 do
Enum.drop(parts, -1) |> Enum.join(".")
else
Enum.join(parts)
end
cond do
content_type == "application/octet-stream" ->
filename
ext = List.first(MIME.extensions(content_type)) ->
new_filename <> "." <> ext
true ->
Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
end
end
defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>) do
"image/png"
end
defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _::binary>>) do
"image/gif"
end
defp check_mime_type(<<0xFF, 0xD8, 0xFF, _::binary>>) do
"image/jpeg"
end
defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _::binary>>) do
"video/webm"
end
defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
"video/mp4"
end
defp check_mime_type(<<0x49, 0x44, 0x33, _::binary>>) do
"audio/mpeg"
end
defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0, _::binary>>) do
"audio/mpeg"
end
defp check_mime_type(
<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::size(160), 0x80, 0x74, 0x68, 0x65,
0x6F, 0x72, 0x61, _::binary>>
) do
"video/ogg"
end
defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary>>) do
"audio/ogg"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
"audio/wav"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
"image/webp"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
"video/avi"
end
defp check_mime_type(_) do
@default
end
end
......@@ -66,6 +66,7 @@ defp get_description(opts, upload) do
end
@spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()}
@doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct."
def store(upload, opts \\ []) do
opts = get_opts(opts)
......@@ -139,14 +140,13 @@ defp get_opts(opts) do
end
defp prepare_upload(%Plug.Upload{} = file, opts) do
with :ok <- check_file_size(file.path, opts.size_limit),
{:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do
with :ok <- check_file_size(file.path, opts.size_limit) do
{:ok,
%__MODULE__{
id: UUID.generate(),
name: name,
name: file.filename,
tempfile: file.path,
content_type: content_type
content_type: file.content_type
}}
end
end
......@@ -154,16 +154,17 @@ defp prepare_upload(%Plug.Upload{} = file, opts) do
defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
data = Base.decode64!(parsed["data"], ignore: :whitespace)
hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))
hash = Base.encode16(:crypto.hash(:sha256, data), lower: true)
with :ok <- check_binary_size(data, opts.size_limit),
tmp_path <- tempfile_for_image(data),
{:ok, content_type, name} <-
Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do
{:ok, %{mime_type: content_type}} <-
Majic.perform({:bytes, data}, pool: Pleroma.MajicPool),
[ext | _] <- MIME.extensions(content_type) do
{:ok,
%__MODULE__{
id: UUID.generate(),
name: name,
name: hash <> "." <> ext,
tempfile: tmp_path,
content_type: content_type
}}
......@@ -172,7 +173,7 @@ defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
# For Mix.Tasks.MigrateLocalUploads
defp prepare_upload(%__MODULE__{tempfile: path} = upload, _opts) do
with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
with {:ok, %{mime_type: content_type}} <- Majic.perform(path, pool: Pleroma.MajicPool) do
{:ok, %__MODULE__{upload | content_type: content_type}}
end
end
......
......@@ -107,7 +107,7 @@ defmodule Pleroma.User do
field(:note_count, :integer, default: 0)
field(:follower_count, :integer, default: 0)
field(:following_count, :integer, default: 0)
field(:locked, :boolean, default: false)
field(:is_locked, :boolean, default: false)
field(:confirmation_pending, :boolean, default: false)
field(:password_reset_pending, :boolean, default: false)
field(:approval_pending, :boolean, default: false)
......@@ -136,7 +136,7 @@ defmodule Pleroma.User do
field(:pleroma_settings_store, :map, default: %{})
field(:fields, {:array, :map}, default: [])
field(:raw_fields, {:array, :map}, default: [])
field(:discoverable, :boolean, default: false)
field(:is_discoverable, :boolean, default: false)
field(:invisible, :boolean, default: false)
field(:allow_following_move, :boolean, default: true)
field(:skip_thread_containment, :boolean, default: false)
......@@ -436,7 +436,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
:avatar,
:ap_enabled,
:banner,
:locked,
:is_locked,
:last_refreshed_at,
:uri,
:follower_address,
......@@ -448,7 +448,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
:follower_count,
:fields,
:following_count,
:discoverable,
:is_discoverable,
:invisible,
:actor_type,
:also_known_as,
......@@ -479,7 +479,7 @@ def update_changeset(struct, params \\ %{}) do
:public_key,
:inbox,
:shared_inbox,
:locked,
:is_locked,
:no_rich_text,
:default_scope,
:banner,
......@@ -495,7 +495,7 @@ def update_changeset(struct, params \\ %{}) do
:fields,
:raw_fields,
:pleroma_settings_store,
:discoverable,