Commit 2437f512 authored by rinpatch's avatar rinpatch
Browse files

Merge branch 'fix/config-gen-releases' into 'develop'

Refactor mix tasks a bit and make config generator work with releases

See merge request !1312
parents 3b48876c e3e8a7dc
Pipeline #13596 passed with stages
in 39 minutes and 39 seconds
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Common do defmodule Mix.Pleroma do
@doc "Common functions to be reused in mix tasks" @doc "Common functions to be reused in mix tasks"
def start_pleroma do def start_pleroma do
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
...@@ -14,7 +14,7 @@ def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do ...@@ -14,7 +14,7 @@ def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
end end
def shell_prompt(prompt, defval \\ nil, defname \\ nil) do def shell_prompt(prompt, defval \\ nil, defname \\ nil) do
prompt_message = "#{prompt} [#{defname || defval}]" prompt_message = "#{prompt} [#{defname || defval}] "
input = input =
if mix_shell?(), if mix_shell?(),
......
defmodule Mix.Tasks.Pleroma.Benchmark do defmodule Mix.Tasks.Pleroma.Benchmark do
import Mix.Pleroma
use Mix.Task use Mix.Task
alias Mix.Tasks.Pleroma.Common
def run(["search"]) do def run(["search"]) do
Common.start_pleroma() start_pleroma()
Benchee.run(%{ Benchee.run(%{
"search" => fn -> "search" => fn ->
...@@ -13,7 +13,7 @@ def run(["search"]) do ...@@ -13,7 +13,7 @@ def run(["search"]) do
end end
def run(["tag"]) do def run(["tag"]) do
Common.start_pleroma() start_pleroma()
Benchee.run(%{ Benchee.run(%{
"tag" => fn -> "tag" => fn ->
......
defmodule Mix.Tasks.Pleroma.Config do defmodule Mix.Tasks.Pleroma.Config do
use Mix.Task use Mix.Task
alias Mix.Tasks.Pleroma.Common import Mix.Pleroma
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.Web.AdminAPI.Config alias Pleroma.Web.AdminAPI.Config
@shortdoc "Manages the location of the config" @shortdoc "Manages the location of the config"
...@@ -17,7 +17,7 @@ defmodule Mix.Tasks.Pleroma.Config do ...@@ -17,7 +17,7 @@ defmodule Mix.Tasks.Pleroma.Config do
""" """
def run(["migrate_to_db"]) do def run(["migrate_to_db"]) do
Common.start_pleroma() start_pleroma()
if Pleroma.Config.get([:instance, :dynamic_configuration]) do if Pleroma.Config.get([:instance, :dynamic_configuration]) do
Application.get_all_env(:pleroma) Application.get_all_env(:pleroma)
...@@ -37,7 +37,7 @@ def run(["migrate_to_db"]) do ...@@ -37,7 +37,7 @@ def run(["migrate_to_db"]) do
end end
def run(["migrate_from_db", env]) do def run(["migrate_from_db", env]) do
Common.start_pleroma() start_pleroma()
if Pleroma.Config.get([:instance, :dynamic_configuration]) do if Pleroma.Config.get([:instance, :dynamic_configuration]) do
config_path = "config/#{env}.exported_from_db.secret.exs" config_path = "config/#{env}.exported_from_db.secret.exs"
......
...@@ -3,12 +3,12 @@ ...@@ -3,12 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Database do defmodule Mix.Tasks.Pleroma.Database do
alias Mix.Tasks.Pleroma.Common
alias Pleroma.Conversation alias Pleroma.Conversation
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
require Logger require Logger
import Mix.Pleroma
use Mix.Task use Mix.Task
@shortdoc "A collection of database related tasks" @shortdoc "A collection of database related tasks"
...@@ -45,7 +45,7 @@ def run(["remove_embedded_objects" | args]) do ...@@ -45,7 +45,7 @@ def run(["remove_embedded_objects" | args]) do
] ]
) )
Common.start_pleroma() start_pleroma()
Logger.info("Removing embedded objects") Logger.info("Removing embedded objects")
Repo.query!( Repo.query!(
...@@ -66,12 +66,12 @@ def run(["remove_embedded_objects" | args]) do ...@@ -66,12 +66,12 @@ def run(["remove_embedded_objects" | args]) do
end end
def run(["bump_all_conversations"]) do def run(["bump_all_conversations"]) do
Common.start_pleroma() start_pleroma()
Conversation.bump_for_all_activities() Conversation.bump_for_all_activities()
end end
def run(["update_users_following_followers_counts"]) do def run(["update_users_following_followers_counts"]) do
Common.start_pleroma() start_pleroma()
users = Repo.all(User) users = Repo.all(User)
Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.remove_duplicated_following/1)
...@@ -89,7 +89,7 @@ def run(["prune_objects" | args]) do ...@@ -89,7 +89,7 @@ def run(["prune_objects" | args]) do
] ]
) )
Common.start_pleroma() start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
......
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
defmodule Mix.Tasks.Pleroma.Instance do defmodule Mix.Tasks.Pleroma.Instance do
use Mix.Task use Mix.Task
alias Mix.Tasks.Pleroma.Common import Mix.Pleroma
@shortdoc "Manages Pleroma instance" @shortdoc "Manages Pleroma instance"
@moduledoc """ @moduledoc """
...@@ -31,6 +31,8 @@ defmodule Mix.Tasks.Pleroma.Instance do ...@@ -31,6 +31,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
- `--dbpass DBPASS` - the password to use for the database connection - `--dbpass DBPASS` - the password to use for the database connection
- `--indexable Y/N` - Allow/disallow indexing site by search engines - `--indexable Y/N` - Allow/disallow indexing site by search engines
- `--db-configurable Y/N` - Allow/disallow configuring instance from admin part - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
- `--uploads-dir` - the directory uploads go in when using a local uploader
- `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
""" """
def run(["gen" | rest]) do def run(["gen" | rest]) do
...@@ -50,7 +52,9 @@ def run(["gen" | rest]) do ...@@ -50,7 +52,9 @@ def run(["gen" | rest]) do
dbuser: :string, dbuser: :string,
dbpass: :string, dbpass: :string,
indexable: :string, indexable: :string,
db_configurable: :string db_configurable: :string,
uploads_dir: :string,
static_dir: :string
], ],
aliases: [ aliases: [
o: :output, o: :output,
...@@ -70,7 +74,7 @@ def run(["gen" | rest]) do ...@@ -70,7 +74,7 @@ def run(["gen" | rest]) do
if proceed? do if proceed? do
[domain, port | _] = [domain, port | _] =
String.split( String.split(
Common.get_option( get_option(
options, options,
:domain, :domain,
"What domain will your instance use? (e.g pleroma.soykaf.com)" "What domain will your instance use? (e.g pleroma.soykaf.com)"
...@@ -79,16 +83,16 @@ def run(["gen" | rest]) do ...@@ -79,16 +83,16 @@ def run(["gen" | rest]) do
) ++ [443] ) ++ [443]
name = name =
Common.get_option( get_option(
options, options,
:instance_name, :instance_name,
"What is the name of your instance? (e.g. Pleroma/Soykaf)" "What is the name of your instance? (e.g. Pleroma/Soykaf)"
) )
email = Common.get_option(options, :admin_email, "What is your admin email address?") email = get_option(options, :admin_email, "What is your admin email address?")
notify_email = notify_email =
Common.get_option( get_option(
options, options,
:notify_email, :notify_email,
"What email address do you want to use for sending email notifications?", "What email address do you want to use for sending email notifications?",
...@@ -96,7 +100,7 @@ def run(["gen" | rest]) do ...@@ -96,7 +100,7 @@ def run(["gen" | rest]) do
) )
indexable = indexable =
Common.get_option( get_option(
options, options,
:indexable, :indexable,
"Do you want search engines to index your site? (y/n)", "Do you want search engines to index your site? (y/n)",
...@@ -104,21 +108,19 @@ def run(["gen" | rest]) do ...@@ -104,21 +108,19 @@ def run(["gen" | rest]) do
) === "y" ) === "y"
db_configurable? = db_configurable? =
Common.get_option( get_option(
options, options,
:db_configurable, :db_configurable,
"Do you want to be able to configure instance from admin part? (y/n)", "Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)",
"y" "y"
) === "y" ) === "y"
dbhost = dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
dbname = dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma_dev")
Common.get_option(options, :dbname, "What is the name of your database?", "pleroma_dev")
dbuser = dbuser =
Common.get_option( get_option(
options, options,
:dbuser, :dbuser,
"What is the user used to connect to your database?", "What is the user used to connect to your database?",
...@@ -126,7 +128,7 @@ def run(["gen" | rest]) do ...@@ -126,7 +128,7 @@ def run(["gen" | rest]) do
) )
dbpass = dbpass =
Common.get_option( get_option(
options, options,
:dbpass, :dbpass,
"What is the password used to connect to your database?", "What is the password used to connect to your database?",
...@@ -134,6 +136,22 @@ def run(["gen" | rest]) do ...@@ -134,6 +136,22 @@ def run(["gen" | rest]) do
"autogenerated" "autogenerated"
) )
uploads_dir =
get_option(
options,
:upload_dir,
"What directory should media uploads go in (when using the local uploader)?",
Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads])
)
static_dir =
get_option(
options,
:static_dir,
"What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
Pleroma.Config.get([:instance, :static_dir])
)
secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64) secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8) signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
{web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1) {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
...@@ -150,12 +168,13 @@ def run(["gen" | rest]) do ...@@ -150,12 +168,13 @@ def run(["gen" | rest]) do
dbname: dbname, dbname: dbname,
dbuser: dbuser, dbuser: dbuser,
dbpass: dbpass, dbpass: dbpass,
version: Pleroma.Mixfile.project() |> Keyword.get(:version),
secret: secret, secret: secret,
signing_salt: signing_salt, signing_salt: signing_salt,
web_push_public_key: Base.url_encode64(web_push_public_key, padding: false), web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
web_push_private_key: Base.url_encode64(web_push_private_key, padding: false), web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
db_configurable?: db_configurable? db_configurable?: db_configurable?,
static_dir: static_dir,
uploads_dir: uploads_dir
) )
result_psql = result_psql =
...@@ -166,31 +185,31 @@ def run(["gen" | rest]) do ...@@ -166,31 +185,31 @@ def run(["gen" | rest]) do
dbpass: dbpass dbpass: dbpass
) )
Common.shell_info( shell_info(
"Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs." "Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs."
) )
File.write(config_path, result_config) File.write(config_path, result_config)
Common.shell_info("Writing #{psql_path}.") shell_info("Writing #{psql_path}.")
File.write(psql_path, result_psql) File.write(psql_path, result_psql)
write_robots_txt(indexable) write_robots_txt(indexable)
Common.shell_info( shell_info(
"\n" <> "\n" <>
""" """
To get started: To get started:
1. Verify the contents of the generated files. 1. Verify the contents of the generated files.
2. Run `sudo -u postgres psql -f #{Common.escape_sh_path(psql_path)}`. 2. Run `sudo -u postgres psql -f #{escape_sh_path(psql_path)}`.
""" <> """ <>
if config_path in ["config/dev.secret.exs", "config/prod.secret.exs"] do if config_path in ["config/dev.secret.exs", "config/prod.secret.exs"] do
"" ""
else else
"3. Run `mv #{Common.escape_sh_path(config_path)} 'config/prod.secret.exs'`." "3. Run `mv #{escape_sh_path(config_path)} 'config/prod.secret.exs'`."
end end
) )
else else
Common.shell_error( shell_error(
"The task would have overwritten the following files:\n" <> "The task would have overwritten the following files:\n" <>
(Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <> (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <>
"Rerun with `--force` to overwrite them." "Rerun with `--force` to overwrite them."
...@@ -215,10 +234,10 @@ defp write_robots_txt(indexable) do ...@@ -215,10 +234,10 @@ defp write_robots_txt(indexable) do
if File.exists?(robots_txt_path) do if File.exists?(robots_txt_path) do
File.cp!(robots_txt_path, "#{robots_txt_path}.bak") File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
Common.shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak") shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak")
end end
File.write(robots_txt_path, robots_txt) File.write(robots_txt_path, robots_txt)
Common.shell_info("Writing #{robots_txt_path}.") shell_info("Writing #{robots_txt_path}.")
end end
end end
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
defmodule Mix.Tasks.Pleroma.Relay do defmodule Mix.Tasks.Pleroma.Relay do
use Mix.Task use Mix.Task
alias Mix.Tasks.Pleroma.Common import Mix.Pleroma
alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Relay
@shortdoc "Manages remote relays" @shortdoc "Manages remote relays"
...@@ -24,24 +24,24 @@ defmodule Mix.Tasks.Pleroma.Relay do ...@@ -24,24 +24,24 @@ defmodule Mix.Tasks.Pleroma.Relay do
Example: ``mix pleroma.relay unfollow https://example.org/relay`` Example: ``mix pleroma.relay unfollow https://example.org/relay``
""" """
def run(["follow", target]) do def run(["follow", target]) do
Common.start_pleroma() start_pleroma()
with {:ok, _activity} <- Relay.follow(target) do with {:ok, _activity} <- Relay.follow(target) do
# put this task to sleep to allow the genserver to push out the messages # put this task to sleep to allow the genserver to push out the messages
:timer.sleep(500) :timer.sleep(500)
else else
{:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}")
end end
end end
def run(["unfollow", target]) do def run(["unfollow", target]) do
Common.start_pleroma() start_pleroma()
with {:ok, _activity} <- Relay.unfollow(target) do with {:ok, _activity} <- Relay.unfollow(target) do
# put this task to sleep to allow the genserver to push out the messages # put this task to sleep to allow the genserver to push out the messages
:timer.sleep(500) :timer.sleep(500)
else else
{:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}")
end end
end end
end end
...@@ -16,7 +16,6 @@ config :pleroma, :instance, ...@@ -16,7 +16,6 @@ config :pleroma, :instance,
notify_email: "<%= notify_email %>", notify_email: "<%= notify_email %>",
limit: 5000, limit: 5000,
registrations_open: true, registrations_open: true,
dedupe_media: false,
dynamic_configuration: <%= db_configurable? %> dynamic_configuration: <%= db_configurable? %>
config :pleroma, :media_proxy, config :pleroma, :media_proxy,
...@@ -38,6 +37,9 @@ config :web_push_encryption, :vapid_details, ...@@ -38,6 +37,9 @@ config :web_push_encryption, :vapid_details,
public_key: "<%= web_push_public_key %>", public_key: "<%= web_push_public_key %>",
private_key: "<%= web_push_private_key %>" private_key: "<%= web_push_private_key %>"
config :pleroma, :instance, static_dir: "<%= static_dir %>"
config :pleroma, Pleroma.Uploaders.Local, uploads: "<%= uploads_dir %>"
# Enable Strict-Transport-Security once SSL is working: # Enable Strict-Transport-Security once SSL is working:
# config :pleroma, :http_security, # config :pleroma, :http_security,
# sts: true # sts: true
......
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
defmodule Mix.Tasks.Pleroma.Uploads do defmodule Mix.Tasks.Pleroma.Uploads do
use Mix.Task use Mix.Task
alias Mix.Tasks.Pleroma.Common import Mix.Pleroma
alias Pleroma.Upload alias Pleroma.Upload
alias Pleroma.Uploaders.Local alias Pleroma.Uploaders.Local
require Logger require Logger
...@@ -24,7 +24,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do ...@@ -24,7 +24,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do
""" """
def run(["migrate_local", target_uploader | args]) do def run(["migrate_local", target_uploader | args]) do
delete? = Enum.member?(args, "--delete") delete? = Enum.member?(args, "--delete")
Common.start_pleroma() start_pleroma()
local_path = Pleroma.Config.get!([Local, :uploads]) local_path = Pleroma.Config.get!([Local, :uploads])
uploader = Module.concat(Pleroma.Uploaders, target_uploader) uploader = Module.concat(Pleroma.Uploaders, target_uploader)
...@@ -38,10 +38,10 @@ def run(["migrate_local", target_uploader | args]) do ...@@ -38,10 +38,10 @@ def run(["migrate_local", target_uploader | args]) do
Pleroma.Config.put([Upload, :uploader], uploader) Pleroma.Config.put([Upload, :uploader], uploader)
end end
Common.shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}") shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}")
if delete? do if delete? do
Common.shell_info( shell_info(
"Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)" "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
) )
...@@ -78,7 +78,7 @@ def run(["migrate_local", target_uploader | args]) do ...@@ -78,7 +78,7 @@ def run(["migrate_local", target_uploader | args]) do
|> Enum.filter(& &1) |> Enum.filter(& &1)
total_count = length(uploads) total_count = length(uploads)
Common.shell_info("Found #{total_count} uploads") shell_info("Found #{total_count} uploads")
uploads uploads
|> Task.async_stream( |> Task.async_stream(
...@@ -90,7 +90,7 @@ def run(["migrate_local", target_uploader | args]) do ...@@ -90,7 +90,7 @@ def run(["migrate_local", target_uploader | args]) do
:ok :ok
error -> error ->
Common.shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
end end
end, end,
timeout: 150_000 timeout: 150_000
...@@ -99,10 +99,10 @@ def run(["migrate_local", target_uploader | args]) do ...@@ -99,10 +99,10 @@ def run(["migrate_local", target_uploader | args]) do
# credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation
|> Enum.reduce(0, fn done, count -> |> Enum.reduce(0, fn done, count ->
count = count + length(done) count = count + length(done)
Common.shell_info("Uploaded #{count}/#{total_count} files") shell_info("Uploaded #{count}/#{total_count} files")
count count
end) end