Merge remote-tracking branch 'remotes/origin/develop' into 1559-follow-request-notifications

# Conflicts:
#	CHANGELOG.md

CHANGELOG.md

@@ -4,9 +4,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
-### Changed
-- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `<br/>` for newlines
-
 ### Removed
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
@@ -15,15 +12,73 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - NodeInfo: `pleroma_emoji_reactions` to the `features` list.
 - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
 - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
+- Mix task to create trusted OAuth App.
 - Notifications: Added `follow_request` notification type (configurable, see `[:notifications, :enable_follow_request_notifications]` setting).
 <details>
   <summary>API Changes</summary>
 - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
+- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
+- Admin API: endpoints for create/update/delete OAuth Apps.
+</details>
+
+### Fixed
+- Support pagination in conversations API
+
+## [unreleased-patch]
+### Fixed
+- Logger configuration through AdminFE
+
+### Added
+<details>
+  <summary>API Changes</summary>
+- Admin API: `GET /api/pleroma/admin/need_reboot`.
 </details>
 
+## [2.0.2] - 2020-04-08
+### Added
+- Support for Funkwhale's `Audio` activity
+- Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials`
+
+### Fixed
+- Blocked/muted users still generating push notifications
+- Input textbox for bio ignoring newlines
+- OTP: Inability to use PostgreSQL databases with SSL
+- `user delete_activities` breaking when trying to delete already deleted posts
+- Incorrect URL for Funkwhale channels
+
+### Upgrade notes
+1. Restart Pleroma
+
+## [2.0.1] - 2020-03-15
+### Security
+- Static-FE: Fix remote posts not being sanitized
+
+### Fixed
+- 500 errors when no `Accept` header is present if Static-FE is enabled
+- Instance panel not being updated immediately due to wrong `Cache-Control` headers
+- Statuses posted with BBCode/Markdown having unncessary newlines in Pleroma-FE
+- OTP: Fix some settings not being migrated to in-database config properly
+- No `Cache-Control` headers on attachment/media proxy requests
+- Character limit enforcement being off by 1
+- Mastodon Streaming API: hashtag timelines not working
+
+### Changed
+- BBCode and Markdown formatters will no longer return any `\n` and only use `<br/>` for newlines
+- Mastodon API: Allow registration without email if email verification is not enabled
+
+### Upgrade notes
+#### Nginx only
+1. Remove `proxy_ignore_headers Cache-Control;` and `proxy_hide_header  Cache-Control;` from your config.
+
+#### Everyone
+1. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+2. Restart Pleroma
+
 ## [2.0.0] - 2019-03-08
 ### Security
-- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
+- Mastodon API: Fix being able to request enormous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
 
 ### Removed
 - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`

benchmarks/load_testing/fetcher.ex

@@ -386,47 +386,56 @@ defp render_timelines(user) do
 
     favourites = ActivityPub.fetch_favourites(user)
 
+    output_relationships =
+      !!Pleroma.Config.get([:extensions, :output_relationships_in_statuses_by_default])
+
     Benchee.run(
       %{
         "Rendering home timeline" => fn ->
           StatusView.render("index.json", %{
             activities: home_activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: !output_relationships
           })
         end,
         "Rendering direct timeline" => fn ->
           StatusView.render("index.json", %{
             activities: direct_activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: !output_relationships
           })
         end,
         "Rendering public timeline" => fn ->
           StatusView.render("index.json", %{
             activities: public_activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: !output_relationships
           })
         end,
         "Rendering tag timeline" => fn ->
           StatusView.render("index.json", %{
             activities: tag_activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: !output_relationships
           })
         end,
         "Rendering notifications" => fn ->
           Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
             notifications: notifications,
-            for: user
+            for: user,
+            skip_relationships: !output_relationships
           })
         end,
         "Rendering favourites timeline" => fn ->
           StatusView.render("index.json", %{
             activities: favourites,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: !output_relationships
           })
         end
       },

config/config.exs

@@ -240,6 +240,8 @@
   extended_nickname_format: true,
   cleanup_attachments: false
 
+config :pleroma, :extensions, output_relationships_in_statuses_by_default: true
+
 config :pleroma, :feed,
   post_title: %{
     max_length: 100,

docs/API/admin_api.md

@@ -392,6 +392,19 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `email`
   - `name`, optional
 
+- Response:
+  - On success: `204`, empty response
+  - On failure:
+    - 400 Bad Request, JSON:
+
+    ```json
+      [
+        {
+          "error": "Appropriate error message here"
+        }
+      ]
+    ```
+
 ## `GET /api/pleroma/admin/users/:nickname/password_reset`
 
 ### Get a password reset token for a given nickname
@@ -773,6 +786,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ### Restarts pleroma application
 
+**Only works when configuration from database is enabled.**
+
 - Params: none
 - Response:
   - On failure:
@@ -782,11 +797,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 {}
 ```
 
+## `GET /api/pleroma/admin/need_reboot`
+
+### Returns the flag whether the pleroma should be restarted
+
+- Params: none
+- Response:
+  - `need_reboot` - boolean
+```json
+{
+  "need_reboot": false
+}
+```
+
 ## `GET /api/pleroma/admin/config`
 
 ### Get list of merged default settings with saved in database.
 
-*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.*
 
 **Only works when configuration from database is enabled.**
 
@@ -808,13 +836,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   "need_reboot": true
 }
 ```
- need_reboot - *optional*, if were changed reboot time settings.
 
 ## `POST /api/pleroma/admin/config`
 
 ### Update config settings
 
-*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.*
 
 **Only works when configuration from database is enabled.**
 
@@ -958,7 +985,6 @@ config :quack,
   "need_reboot": true
 }
 ```
-need_reboot - *optional*, if were changed reboot time settings.
 
 ## ` GET /api/pleroma/admin/config/descriptions`
 
@@ -1062,3 +1088,104 @@ Loads json generated from `config/descriptions.exs`.
   }
 }
 ```
+
+## `GET /api/pleroma/admin/oauth_app`
+
+### List OAuth app
+
+- Params:
+  - *optional* `name`
+  - *optional* `client_id`
+  - *optional* `page`
+  - *optional* `page_size`
+  - *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "apps": [
+    {
+      "id": 1,
+      "name": "App name",
+      "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+      "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+      "redirect_uri": "https://example.com/oauth-callback",
+      "website": "https://example.com",
+      "trusted": true
+    }
+  ],
+  "count": 17,
+  "page_size": 50
+}
+```
+
+
+## `POST /api/pleroma/admin/oauth_app`
+
+### Create OAuth App
+
+- Params:
+  - `name`
+  - `redirect_uris`
+  - `scopes`
+  - *optional* `website`
+  - *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "id": 1,
+  "name": "App name",
+  "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+  "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+  "redirect_uri": "https://example.com/oauth-callback",
+  "website": "https://example.com",
+  "trusted": true
+}
+```
+
+- On failure:
+```json
+{
+  "redirect_uris": "can't be blank",
+  "name": "can't be blank"
+}
+```
+
+## `PATCH /api/pleroma/admin/oauth_app/:id`
+
+### Update OAuth App
+
+- Params:
+  -  *optional* `name`
+  -  *optional* `redirect_uris`
+  -  *optional* `scopes`
+  -  *optional* `website`
+  -  *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "id": 1,
+  "name": "App name",
+  "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+  "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+  "redirect_uri": "https://example.com/oauth-callback",
+  "website": "https://example.com",
+  "trusted": true
+}
+```
+
+## `DELETE /api/pleroma/admin/oauth_app/:id`
+
+### Delete OAuth App
+
+- Params: None
+
+- Response:
+  - On success: `204`, empty response
+  - On failure:
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing
\ No newline at end of file

docs/administration/CLI_tasks/oauth_app.md

+# Creating trusted OAuth App
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Create trusted OAuth App.
+
+Optional params:
+  * `-s SCOPES` - scopes for app, e.g. `read,write,follow,push`.
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl app create -n APP_NAME -r REDIRECT_URI
+```
+
+```sh tab="From Source"
+mix pleroma.app create -n APP_NAME -r REDIRECT_URI
+```
\ No newline at end of file

docs/configuration/mrf.md

@@ -113,7 +113,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do
 
   @impl true
   def describe do
-    {:ok, %{mrf_sample: %{content: "new message content"}}}`
+    {:ok, %{mrf_sample: %{content: "new message content"}}}
   end
 end
 ```

lib/mix/pleroma.ex

@@ -5,7 +5,6 @@
 defmodule Mix.Pleroma do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
-    Mix.Task.run("app.start")
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do

lib/mix/tasks/pleroma/app.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.App do
+  @moduledoc File.read!("docs/administration/CLI_tasks/oauth_app.md")
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Creates trusted OAuth App"
+
+  def run(["create" | options]) do
+    start_pleroma()
+
+    {opts, _} =
+      OptionParser.parse!(options,
+        strict: [name: :string, redirect_uri: :string, scopes: :string],
+        aliases: [n: :name, r: :redirect_uri, s: :scopes]
+      )
+
+    scopes =
+      if opts[:scopes] do
+        String.split(opts[:scopes], ",")
+      else
+        ["read", "write", "follow", "push"]
+      end
+
+    params = %{
+      client_name: opts[:name],
+      redirect_uris: opts[:redirect_uri],
+      trusted: true,
+      scopes: scopes
+    }
+
+    with {:ok, app} <- Pleroma.Web.OAuth.App.create(params) do
+      shell_info("#{app.client_name} successfully created:")
+      shell_info("App client_id: " <> app.client_id)
+      shell_info("App client_secret: " <> app.client_secret)
+    else
+      {:error, changeset} ->
+        shell_error("Creating failed:")
+
+        Enum.each(Pleroma.Web.OAuth.App.errors(changeset), fn {key, error} ->
+          shell_error("#{key}: #{error}")
+        end)
+    end
+  end
+end

lib/mix/tasks/pleroma/benchmark.ex

@@ -67,7 +67,8 @@ def run(["render_timeline", nickname | _] = args) do
           Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
             activities: activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: true
           })
         end
       },

lib/pleroma/config/transfer_task.ex

@@ -54,10 +54,19 @@ def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
           [:pleroma, nil, :prometheus]
         end
 
+      {logger, other} =
+        (Repo.all(ConfigDB) ++ deleted_settings)
+        |> Enum.map(&transform_and_merge/1)
+        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
+
+      logger
+      |> Enum.sort()
+      |> Enum.each(&configure/1)
+
       started_applications = Application.started_applications()
 
-      (Repo.all(ConfigDB) ++ deleted_settings)
-      |> Enum.map(&merge_and_update/1)
+      other
+      |> Enum.map(&update/1)
       |> Enum.uniq()
       |> Enum.reject(&(&1 in reject_restart))
       |> maybe_set_pleroma_last()
@@ -81,51 +90,71 @@ defp maybe_set_pleroma_last(apps) do
     end
   end
 
-  defp group_for_restart(:logger, key, _, merged_value) do
-    # change logger configuration in runtime, without restart
-    if Keyword.keyword?(merged_value) and
-         key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
-      Logger.configure_backend(key, merged_value)
-    else
-      Logger.configure([{key, merged_value}])
-    end
+  defp transform_and_merge(%{group: group, key: key, value: value} = setting) do
+    group = ConfigDB.from_string(group)
+    key = ConfigDB.from_string(key)
+    value = ConfigDB.from_binary(value)
 
-    nil
-  end
+    default = Config.Holder.default_config(group, key)
 
-  defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
+    merged =
+      cond do
+        Ecto.get_meta(setting, :state) == :deleted -> default
+        can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
+        true -> value
+      end
 
-  defp group_for_restart(group, key, value, _) do
-    if pleroma_need_restart?(group, key, value), do: group
+    {group, key, value, merged}
   end
 
-  defp merge_and_update(setting) do
-    try do
-      key = ConfigDB.from_string(setting.key)
-      group = ConfigDB.from_string(setting.group)
+  # change logger configuration in runtime, without restart
+  defp configure({:quack, key, _, merged}) do
+    Logger.configure_backend(Quack.Logger, [{key, merged}])
+    :ok = update_env(:quack, key, merged)
+  end
 
-      default = Config.Holder.default_config(group, key)
-      value = ConfigDB.from_binary(setting.value)
+  defp configure({_, :backends, _, merged}) do
+    # removing current backends
+    Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
 
-      merged_value =
-        cond do
-          Ecto.get_meta(setting, :state) == :deleted -> default
-          can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
-          true -> value
-        end
+    Enum.each(merged, &Logger.add_backend/1)
 
-      :ok = update_env(group, key, merged_value)
+    :ok = update_env(:logger, :backends, merged)
+  end
+
+  defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
+    merged =
+      if key == :console do
+        put_in(merged[:format], merged[:format] <> "\n")
+      else
+        merged
+      end
+
+    backend =
+      if key == :ex_syslogger,
+        do: {ExSyslogger, :ex_syslogger},
+        else: key
+
+    Logger.configure_backend(backend, merged)
+    :ok = update_env(:logger, key, merged)
+  end
 
-      group_for_restart(group, key, value, merged_value)
+  defp configure({_, key, _, merged}) do
+    Logger.configure([{key, merged}])
+    :ok = update_env(:logger, key, merged)
+  end
+
+  defp update({group, key, value, merged}) do
+    try do
+      :ok = update_env(group, key, merged)
+
+      if group != :pleroma or pleroma_need_restart?(group, key, value), do: group
     rescue
       error ->
         error_msg =
-          "updating env causes error, group: " <>
-            inspect(setting.group) <>
-            " key: " <>
-            inspect(setting.key) <>
-            " value: " <>
-            inspect(ConfigDB.from_binary(setting.value)) <> " error: " <> inspect(error)
+          "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{
+            inspect(value)
+          } error: #{inspect(error)}"
 
         Logger.warn(error_msg)
 
@@ -133,6 +162,9 @@ defp merge_and_update(setting) do
     end
   end
 
+  defp update_env(group, key, nil), do: Application.delete_env(group, key)
+  defp update_env(group, key, value), do: Application.put_env(group, key, value)
+
   @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
   def pleroma_need_restart?(group, key, value) do
     group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
@@ -150,9 +182,6 @@ defp group_and_subkey_need_reboot?(group, key, value) do
       end)
   end
 
-  defp update_env(group, key, nil), do: Application.delete_env(group, key)
-  defp update_env(group, key, value), do: Application.put_env(group, key, value)
-
   defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
 
   defp restart(started_applications, app, _) do

lib/pleroma/ecto_enums.ex

@@ -4,10 +4,16 @@
 
 import EctoEnum
 
-defenum(UserRelationshipTypeEnum,
+defenum(Pleroma.UserRelationship.Type,
   block: 1,
   mute: 2,
   reblog_mute: 3,
   notification_mute: 4,
   inverse_subscription: 5
 )
+
+defenum(Pleroma.FollowingRelationship.State,
+  follow_pending: 1,
+  follow_accept: 2,
+  follow_reject: 3
+)

lib/pleroma/emoji/formatter.ex

@@ -38,22 +38,14 @@ def demojify(text) do
 
   def demojify(text, nil), do: text
 
-  @doc "Outputs a list of the emoji-shortcodes in a text"
-  def get_emoji(text) when is_binary(text) do
-    Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
-      String.contains?(text, ":#{emoji}:")
-    end)
-  end
-
-  def get_emoji(_), do: []
-
   @doc "Outputs a list of the emoji-Maps in a text"