Merge branch 'admin-config' into 'develop'

Admin config

See merge request !1328

docs/api/admin_api.md

@@ -568,8 +568,9 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 {
   configs: [
     {
+      "group": string,
       "key": string,
-      "value": string or {} or []
+      "value": string or {} or [] or {"tuple": []}
      }
   ]
 }
@@ -580,6 +581,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`.
 Atom or boolean value can be passed with `:` in the beginning, e.g. `":true"`, `":upload"`.
 Integer with `i:`, e.g. `"i:150"`.
+Tuple with more than 2 values with `{"tuple": ["first_val", Pleroma.Module, []]}`.
+`{"tuple": ["some_string", "Pleroma.Some.Module", []]}` will be converted to `{"some_string", Pleroma.Some.Module, []}`.
 
 Compile time settings (need instance reboot):
 - all settings by this keys:
@@ -595,8 +598,9 @@ Compile time settings (need instance reboot):
 - Method `POST`
 - Params:
   - `configs` => [
+    - `group` (string)
     - `key` (string)
-    - `value` (string, [], {})
+    - `value` (string, [], {} or {"tuple": []})
     - `delete` = true (optional, if parameter must be deleted)
   ]
 
@@ -606,6 +610,7 @@ Compile time settings (need instance reboot):
 {
   configs: [
     {
+      "group": "pleroma",
       "key": "Pleroma.Upload",
       "value": {
         "uploader": "Pleroma.Uploaders.Local",
@@ -619,6 +624,9 @@ Compile time settings (need instance reboot):
             "follow_redirect": ":true",
             "pool": ":upload"
           }
+        },
+        "dispatch": {
+          "tuple": ["/api/v1/streaming", "Pleroma.Web.MastodonAPI.WebsocketHandler", []]
         }
       }
      }
@@ -631,8 +639,9 @@ Compile time settings (need instance reboot):
 {
   configs: [
     {
+      "group": string,
       "key": string,
-      "value": string or {} or []
+      "value": string or {} or [] or {"tuple": []}
      }
   ]
 }

lib/mix/tasks/pleroma/config.ex

@@ -24,7 +24,7 @@ def run(["migrate_to_db"]) do
       |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
       |> Enum.each(fn {k, v} ->
         key = to_string(k) |> String.replace("Elixir.", "")
-        {:ok, _} = Config.update_or_create(%{key: key, value: v})
+        {:ok, _} = Config.update_or_create(%{group: "pleroma", key: key, value: v})
         Mix.shell().info("#{key} is migrated.")
       end)
 
@@ -51,7 +51,9 @@ def run(["migrate_from_db", env]) do
 
         IO.write(
           file,
-          "config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n"
+          "config :#{config.group}, #{config.key}#{mark} #{
+            inspect(Config.from_binary(config.value))
+          }\r\n"
         )
 
         {:ok, _} = Repo.delete(config)

lib/pleroma/config/transfer_task.ex

@@ -11,8 +11,17 @@ def start_link do
   def load_and_update_env do
     if Pleroma.Config.get([:instance, :dynamic_configuration]) and
          Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do
-      Pleroma.Repo.all(Config)
-      |> Enum.each(&update_env(&1))
+      for_restart =
+        Pleroma.Repo.all(Config)
+        |> Enum.map(&update_env(&1))
+
+      # We need to restart applications for loaded settings take effect
+      for_restart
+      |> Enum.reject(&(&1 in [:pleroma, :ok]))
+      |> Enum.each(fn app ->
+        Application.stop(app)
+        :ok = Application.start(app)
+      end)
     end
   end
 
@@ -25,11 +34,15 @@ defp update_env(setting) do
           setting.key
         end
 
+      group = String.to_existing_atom(setting.group)
+
       Application.put_env(
-        :pleroma,
+        group,
         String.to_existing_atom(key),
         Config.from_binary(setting.value)
       )
+
+      group
     rescue
       e ->
         require Logger

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -377,12 +377,12 @@ def config_update(conn, %{"configs" => configs}) do
       if Pleroma.Config.get([:instance, :dynamic_configuration]) do
         updated =
           Enum.map(configs, fn
-            %{"key" => key, "value" => value} ->
-              {:ok, config} = Config.update_or_create(%{key: key, value: value})
+            %{"group" => group, "key" => key, "value" => value} ->
+              {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})
               config
 
-            %{"key" => key, "delete" => "true"} ->
-              {:ok, _} = Config.delete(key)
+            %{"group" => group, "key" => key, "delete" => "true"} ->
+              {:ok, _} = Config.delete(%{group: group, key: key})
               nil
           end)
           |> Enum.reject(&is_nil(&1))

lib/pleroma/web/admin_api/config.ex

@@ -12,26 +12,27 @@ defmodule Pleroma.Web.AdminAPI.Config do
 
   schema "config" do
     field(:key, :string)
+    field(:group, :string)
     field(:value, :binary)
 
     timestamps()
   end
 
-  @spec get_by_key(String.t()) :: Config.t() | nil
-  def get_by_key(key), do: Repo.get_by(Config, key: key)
+  @spec get_by_params(map()) :: Config.t() | nil
+  def get_by_params(params), do: Repo.get_by(Config, params)
 
   @spec changeset(Config.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do
     config
-    |> cast(params, [:key, :value])
-    |> validate_required([:key, :value])
-    |> unique_constraint(:key)
+    |> cast(params, [:key, :group, :value])
+    |> validate_required([:key, :group, :value])
+    |> unique_constraint(:key, name: :config_group_key_index)
   end
 
   @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
-  def create(%{key: key, value: value}) do
+  def create(params) do
     %Config{}
-    |> changeset(%{key: key, value: transform(value)})
+    |> changeset(Map.put(params, :value, transform(params[:value])))
     |> Repo.insert()
   end
 
@@ -43,20 +44,20 @@ def update(%Config{} = config, %{value: value}) do
   end
 
   @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
-  def update_or_create(%{key: key} = params) do
-    with %Config{} = config <- Config.get_by_key(key) do
+  def update_or_create(params) do
+    with %Config{} = config <- Config.get_by_params(Map.take(params, [:group, :key])) do
       Config.update(config, params)
     else
       nil -> Config.create(params)
     end
   end
 
-  @spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()}
-  def delete(key) do
-    with %Config{} = config <- Config.get_by_key(key) do
+  @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def delete(params) do
+    with %Config{} = config <- Config.get_by_params(params) do
       Repo.delete(config)
     else
-      nil -> {:error, "Config with key #{key} not found"}
+      nil -> {:error, "Config with params #{inspect(params)} not found"}
     end
   end
 
@@ -77,10 +78,21 @@ defp do_convert(values) when is_list(values), do: for(val <- values, do: do_conv
   defp do_convert({k, v} = value) when is_tuple(value),
     do: %{k => do_convert(v)}
 
-  defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value),
-    do: value
+  defp do_convert(value) when is_tuple(value), do: %{"tuple" => do_convert(Tuple.to_list(value))}
+
+  defp do_convert(value) when is_binary(value) or is_map(value) or is_number(value), do: value
+
+  defp do_convert(value) when is_atom(value) do
+    string = to_string(value)
+
+    if String.starts_with?(string, "Elixir."),
+      do: String.trim_leading(string, "Elixir."),
+      else: value
+  end
 
   @spec transform(any()) :: binary()
+  def transform(%{"tuple" => _} = entity), do: :erlang.term_to_binary(do_transform(entity))
+
   def transform(entity) when is_map(entity) do
     tuples =
       for {k, v} <- entity,
@@ -101,11 +113,16 @@ def transform(entity), do: :erlang.term_to_binary(entity)
 
   defp do_transform(%Regex{} = value) when is_map(value), do: value
 
+  defp do_transform(%{"tuple" => [k, values] = entity}) when length(entity) == 2 do
+    {do_transform(k), do_transform(values)}
+  end
+
+  defp do_transform(%{"tuple" => values}) do
+    Enum.reduce(values, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
+  end
+
   defp do_transform(value) when is_map(value) do
-    values =
-      for {key, val} <- value,
-          into: [],
-          do: {String.to_atom(key), do_transform(val)}
+    values = for {key, val} <- value, into: [], do: {String.to_atom(key), do_transform(val)}
 
     Enum.sort(values)
   end
@@ -117,28 +134,27 @@ defp do_transform(value) when is_list(value) do
   defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity)
 
   defp do_transform(value) when is_binary(value) do
-    value = String.trim(value)
+    String.trim(value)
+    |> do_transform_string()
+  end
 
-    case String.length(value) do
-      0 ->
-        nil
+  defp do_transform(value), do: value
 
-      _ ->
-        cond do
-          String.starts_with?(value, "Pleroma") ->
-            String.to_existing_atom("Elixir." <> value)
+  defp do_transform_string(value) when byte_size(value) == 0, do: nil
 
-          String.starts_with?(value, ":") ->
-            String.replace(value, ":", "") |> String.to_existing_atom()
+  defp do_transform_string(value) do
+    cond do
+      String.starts_with?(value, "Pleroma") or String.starts_with?(value, "Phoenix") ->
+        String.to_existing_atom("Elixir." <> value)
 
-          String.starts_with?(value, "i:") ->
-            String.replace(value, "i:", "") |> String.to_integer()
+      String.starts_with?(value, ":") ->
+        String.replace(value, ":", "") |> String.to_existing_atom()
 
-          true ->
-            value
-        end
+      String.starts_with?(value, "i:") ->
+        String.replace(value, "i:", "") |> String.to_integer()
+
+      true ->
+        value
     end
   end
-
-  defp do_transform(value), do: value
 end

lib/pleroma/web/admin_api/views/config_view.ex

@@ -10,6 +10,7 @@ def render("index.json", %{configs: configs}) do
   def render("show.json", %{config: config}) do
     %{
       key: config.key,
+      group: config.group,
       value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value)
     }
   end

priv/repo/migrations/20190622151019_add_group_key_to_config.exs

+defmodule Pleroma.Repo.Migrations.AddGroupKeyToConfig do
+  use Ecto.Migration
+
+  def change do
+    alter table("config") do
+      add(:group, :string)
+    end
+
+    drop(unique_index("config", :key))
+    create(unique_index("config", [:group, :key]))
+  end
+end

test/config/transfer_task_test.exs

@@ -13,19 +13,37 @@ defmodule Pleroma.Config.TransferTaskTest do
 
   test "transfer config values from db to env" do
     refute Application.get_env(:pleroma, :test_key)
-    Pleroma.Web.AdminAPI.Config.create(%{key: "test_key", value: [live: 2, com: 3]})
+    refute Application.get_env(:idna, :test_key)
+
+    Pleroma.Web.AdminAPI.Config.create(%{
+      group: "pleroma",
+      key: "test_key",
+      value: [live: 2, com: 3]
+    })
+
+    Pleroma.Web.AdminAPI.Config.create(%{
+      group: "idna",
+      key: "test_key",
+      value: [live: 15, com: 35]
+    })
 
     Pleroma.Config.TransferTask.start_link()
 
     assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
+    assert Application.get_env(:idna, :test_key) == [live: 15, com: 35]
 
     on_exit(fn ->
       Application.delete_env(:pleroma, :test_key)
+      Application.delete_env(:idna, :test_key)
     end)
   end
 
   test "non existing atom" do
-    Pleroma.Web.AdminAPI.Config.create(%{key: "undefined_atom_key", value: [live: 2, com: 3]})
+    Pleroma.Web.AdminAPI.Config.create(%{
+      group: "pleroma",
+      key: "undefined_atom_key",
+      value: [live: 2, com: 3]
+    })
 
     assert ExUnit.CaptureLog.capture_log(fn ->
              Pleroma.Config.TransferTask.start_link()

test/support/factory.ex

@@ -314,6 +314,7 @@ def registration_factory do
   def config_factory do
     %Pleroma.Web.AdminAPI.Config{
       key: sequence(:key, &"some_key_#{&1}"),
+      group: "pleroma",
       value:
         sequence(
           :value,

test/tasks/config_test.exs

@@ -30,17 +30,26 @@ test "settings are migrated to db" do
 
     Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
 
-    first_db = Config.get_by_key("first_setting")
-    second_db = Config.get_by_key("second_setting")
-    refute Config.get_by_key("Pleroma.Repo")
+    first_db = Config.get_by_params(%{group: "pleroma", key: "first_setting"})
+    second_db = Config.get_by_params(%{group: "pleroma", key: "second_setting"})
+    refute Config.get_by_params(%{group: "pleroma", key: "Pleroma.Repo"})
 
     assert Config.from_binary(first_db.value) == [key: "value", key2: [Pleroma.Repo]]
     assert Config.from_binary(second_db.value) == [key: "value2", key2: [Pleroma.Activity]]
   end
 
   test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
-    Config.create(%{key: "setting_first", value: [key: "value", key2: [Pleroma.Activity]]})
-    Config.create(%{key: "setting_second", value: [key: "valu2", key2: [Pleroma.Repo]]})
+    Config.create(%{
+      group: "pleroma",
+      key: "setting_first",
+      value: [key: "value", key2: [Pleroma.Activity]]
+    })
+
+    Config.create(%{
+      group: "pleroma",
+      key: "setting_second",
+      value: [key: "valu2", key2: [Pleroma.Repo]]
+    })
 
     Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "temp"])
 

test/web/activity_pub/mrf/anti_link_spam_policy_test.exs

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  import ExUnit.CaptureLog
 
   alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy
 
@@ -114,7 +115,9 @@ test "it rejects posts without links" do
         @linkless_message
         |> Map.put("actor", "http://invalid.actor")
 
-      {:reject, _} = AntiLinkSpamPolicy.filter(message)
+      assert capture_log(fn ->
+               {:reject, _} = AntiLinkSpamPolicy.filter(message)
+             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
     end
 
     test "it rejects posts with links" do
@@ -122,7 +125,9 @@ test "it rejects posts with links" do
         @linkful_message
         |> Map.put("actor", "http://invalid.actor")
 
-      {:reject, _} = AntiLinkSpamPolicy.filter(message)
+      assert capture_log(fn ->
+               {:reject, _} = AntiLinkSpamPolicy.filter(message)
+             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
     end
   end
 

test/web/admin_api/admin_api_controller_test.exs

@@ -1343,6 +1343,8 @@ test "with settings in db", %{conn: conn} do
         Application.delete_env(:pleroma, :key4)
         Application.delete_env(:pleroma, :keyaa1)
         Application.delete_env(:pleroma, :keyaa2)
+        Application.delete_env(:pleroma, Pleroma.Web.Endpoint.NotReal)
+        Application.delete_env(:pleroma, Pleroma.Captcha.NotReal)
         :ok = File.rm(temp_file)
       end)
 
@@ -1361,8 +1363,9 @@ test "create new config setting in db", %{conn: conn} do
       conn =
         post(conn, "/api/pleroma/admin/config", %{
           configs: [
-            %{key: "key1", value: "value1"},
+            %{group: "pleroma", key: "key1", value: "value1"},
             %{
+              group: "pleroma",
               key: "key2",
               value: %{
                 "nested_1" => "nested_value1",
@@ -1373,6 +1376,7 @@ test "create new config setting in db", %{conn: conn} do
               }
             },
             %{
+              group: "pleroma",
               key: "key3",
               value: [
                 %{"nested_3" => ":nested_3", "nested_33" => "nested_33"},
@@ -1380,8 +1384,14 @@ test "create new config setting in db", %{conn: conn} do
               ]
             },
             %{
+              group: "pleroma",
               key: "key4",
               value: %{"nested_5" => ":upload", "endpoint" => "https://example.com"}
+            },
+            %{
+              group: "idna",
+              key: "key5",
+              value: %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]}
             }
           ]
         })
@@ -1389,10 +1399,12 @@ test "create new config setting in db", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "configs" => [
                  %{
+                   "group" => "pleroma",
                    "key" => "key1",
                    "value" => "value1"
                  },
                  %{
+                   "group" => "pleroma",
                    "key" => "key2",
                    "value" => [
                      %{"nested_1" => "nested_value1"},
@@ -1405,6 +1417,7 @@ test "create new config setting in db", %{conn: conn} do
                    ]
                  },
                  %{
+                   "group" => "pleroma",
                    "key" => "key3",
                    "value" => [
                      [%{"nested_3" => "nested_3"}, %{"nested_33" => "nested_33"}],
@@ -1412,8 +1425,14 @@ test "create new config setting in db", %{conn: conn} do
                    ]
                  },
                  %{
+                   "group" => "pleroma",
                    "key" => "key4",
                    "value" => [%{"endpoint" => "https://example.com"}, %{"nested_5" => "upload"}]
+                 },
+                 %{
+                   "group" => "idna",
+                   "key" => "key5",
+                   "value" => %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]}
                  }
                ]
              }
@@ -1437,6 +1456,8 @@ test "create new config setting in db", %{conn: conn} do
                endpoint: "https://example.com",
                nested_5: :upload
              ]
+
+      assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []}
     end
 
     test "update config setting & delete", %{conn: conn} do
@@ -1446,14 +1467,15 @@ test "update config setting & delete", %{conn: conn} do
       conn =
         post(conn, "/api/pleroma/admin/config", %{
           configs: [
-            %{key: config1.key, value: "another_value"},
-            %{key: config2.key, delete: "true"}
+            %{group: config1.group, key: config1.key, value: "another_value"},
+            %{group: config2.group, key: config2.key, delete: "true"}
           ]
         })
 
       assert json_response(conn, 200) == %{
                "configs" => [
                  %{
+                   "group" => "pleroma",
                    "key" => config1.key,
                    "value" => "another_value"
                  }
@@ -1463,5 +1485,152 @@ test "update config setting & delete", %{conn: conn} do
       assert Application.get_env(:pleroma, :keyaa1) == "another_value"
       refute Application.get_env(:pleroma, :keyaa2)
     end
+
+    test "common config example", %{conn: conn} do
+      conn =
+        post(conn, "/api/pleroma/admin/config", %{
+          configs: [
+            %{
+              "group" => "pleroma",
+              "key" => "Pleroma.Captcha.NotReal",
+              "value" => %{
+                "enabled" => ":false",
+                "method" => "Pleroma.Captcha.Kocaptcha",
+                "seconds_valid" => "i:60"
+              }
+            }
+          ]
+        })
+
+      assert json_response(conn, 200) == %{
+               "configs" => [
+                 %{
+                   "group" => "pleroma",
+                   "key" => "Pleroma.Captcha.NotReal",
+                   "value" => [
+                     %{"enabled" => false},
+                     %{"method" => "Pleroma.Captcha.Kocaptcha"},
+                     %{"seconds_valid" => 60}
+                   ]
+                 }
+               ]
+             }
+    end
+
+    test "tuples with more than two values", %{conn: conn} do
+      conn =
+        post(conn, "/api/pleroma/admin/config", %{
+          configs: [
+            %{
+              "group" => "pleroma",
+              "key" => "Pleroma.Web.Endpoint.NotReal",
+              "value" => [
+                %{
+                  "http" => %{
+                    "dispatch" => [
+                      %{
+                        "tuple" => [
+                          ":_",
+                          [
+                            %{
+                              "tuple" => [
+                                "/api/v1/streaming",
+                                "Pleroma.Web.MastodonAPI.WebsocketHandler",
+                                []
+                              ]
+                            },
+                            %{
+                              "tuple" => [
+                                "/websocket",
+                                "Phoenix.Endpoint.CowboyWebSocket",
+                                %{
+                                  "tuple" => [
+                                    "Phoenix.Transports.WebSocket",
+                                    %{
+                                      "tuple" => [
+                                        "Pleroma.Web.Endpoint",
+                                        "Pleroma.Web.UserSocket",
+                                        []
+                                      ]
+                                    }
+                                  ]
+                                }
+