Merge branch 'feature/account-export' into 'develop'

Add account export

Closes #847

See merge request !2918

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Experimental websocket-based federation between Pleroma instances.
 - Support pagination of blocks and mutes
 - App metrics: ability to restrict access to specified IP whitelist.
+- Account backup
 - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 
 ### Changed

config/config.exs

@@ -551,6 +551,7 @@
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
@@ -835,6 +836,11 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.User.Backup,
+  purge_after_days: 30,
+  limit_days: 7,
+  dir: nil
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -2297,6 +2297,12 @@
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
@@ -3732,6 +3738,26 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.User.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
+  },
   %{
     group: :prometheus,
     key: Pleroma.Web.Endpoint.MetricsExporter,

docs/API/pleroma_api.md

@@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/v1/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/v1/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

docs/configuration/cheatsheet.md

@@ -1078,6 +1078,20 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Pleroma.User.Backup
+
+!!! note
+    Requires enabled email
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.

lib/pleroma/emails/user_email.ex

@@ -189,4 +189,30 @@ def unsubscribe_url(user, notifications_type) do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      end
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end

lib/pleroma/moderation_log.ex

@@ -655,6 +655,16 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/user/backup.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+  import Pleroma.Web.Gettext
+
+  require Pleroma.Constants
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
+
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user, admin_id \\ nil) do
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user, admin_id),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      BackupWorker.process(backup, admin_id)
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
+      Repo.delete(backup)
+    end
+  end
+
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, dgettext("errors", "Backups require enabled email")}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- export(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
+
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def export(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = dir(name)
+
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
+      {:ok, to_string(zip_path)}
+    end
+  end
+
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
+  def upload(%__MODULE__{} = backup, zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    upload = %Pleroma.Upload{
+      name: backup.file_name,
+      tempfile: zip_path,
+      content_type: backup.content_type,
+      path: Path.join("backups", backup.file_name)
+    }
+
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
+      {:ok, upload}
+    end
+  end
+
+  defp actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(Path.join(dir, "actor.json"), json)
+    end
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = Path.join(dir, "#{name}.json")
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  defp bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
+  end
+
+  defp likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
+  end
+
+  defp statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:actor_id, user.ap_id)
+
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        {:ok, Map.delete(activity, "@context")}
+      end
+    end)
+  end
+end

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -26,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:right_get, :show_user_credentials]
+    when action in [:right_get, :show_user_credentials, :create_backup]
   )
 
   plug(
@@ -441,6 +441,15 @@ def stats(conn, params) do
     json(conn, %{"status_visibility" => counters})
   end
 
+  def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname),
+         {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
+      ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
+
+      json(conn, "")
+    end
+  end
+
   defp page_params(params) do
     {
       fetch_integer_param(params, "page", 1),

lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "List backups",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.index",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "Create a backup",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.create",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp backup do
+    %Schema{
+      title: "Backup",
+      description: "Response schema for a backup",
+      type: :object,
+      properties: %{
+        inserted_at: %Schema{type: :string, format: :"date-time"},
+        content_type: %Schema{type: :string},
+        file_name: %Schema{type: :string},
+        file_size: %Schema{type: :integer},
+        processed: %Schema{type: :boolean}
+      },
+      example: %{
+        "content_type" => "application/zip",
+        "file_name" =>
+          "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
+        "file_size" => 4105,
+        "inserted_at" => "2020-09-08T16:42:07.000Z",
+        "processed" => true
+      }
+    }
+  end
+end

lib/pleroma/web/pleroma_api/controllers/backup_controller.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.User.Backup
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+  plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
+  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
+
+  def index(%{assigns: %{user: user}} = conn, _params) do
+    backups = Backup.list(user)
+    render(conn, "index.json", backups: backups)
+  end
+
+  def create(%{assigns: %{user: user}} = conn, _params) do
+    with {:ok, _} <- Backup.create(user) do
+      backups = Backup.list(user)
+      render(conn, "index.json", backups: backups)
+    end
+  end
+end

lib/pleroma/web/pleroma_api/views/backup_view.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.User.Backup
+  alias Pleroma.Web.CommonAPI.Utils
+
+  def render("show.json", %{backup: %Backup{} = backup}) do
+    %{
+      content_type: backup.content_type,
+      url: download_url(backup),
+      file_size: backup.file_size,
+      processed: backup.processed,
+      inserted_at: Utils.to_masto_date(backup.inserted_at)
+    }
+  end