backup.ex 6.93 KB
Newer Older
minibikini's avatar
minibikini committed
1
2
3
4
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only

5
defmodule Pleroma.User.Backup do
minibikini's avatar
minibikini committed
6
7
8
9
  use Ecto.Schema

  import Ecto.Changeset
  import Ecto.Query
minibikini's avatar
minibikini committed
10
  import Pleroma.Web.Gettext
minibikini's avatar
minibikini committed
11

minibikini's avatar
minibikini committed
12
13
  require Pleroma.Constants

minibikini's avatar
minibikini committed
14
15
  alias Pleroma.Activity
  alias Pleroma.Bookmark
minibikini's avatar
minibikini committed
16
  alias Pleroma.Repo
minibikini's avatar
minibikini committed
17
18
19
20
  alias Pleroma.User
  alias Pleroma.Web.ActivityPub.ActivityPub
  alias Pleroma.Web.ActivityPub.Transmogrifier
  alias Pleroma.Web.ActivityPub.UserView
minibikini's avatar
minibikini committed
21
  alias Pleroma.Workers.BackupWorker
minibikini's avatar
minibikini committed
22

minibikini's avatar
minibikini committed
23
24
25
26
27
28
29
30
31
32
33
  schema "backups" do
    field(:content_type, :string)
    field(:file_name, :string)
    field(:file_size, :integer, default: 0)
    field(:processed, :boolean, default: false)

    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)

    timestamps()
  end

minibikini's avatar
minibikini committed
34
  def create(user, admin_id \\ nil) do
minibikini's avatar
minibikini committed
35
36
    with :ok <- validate_email_enabled(),
         :ok <- validate_user_email(user),
minibikini's avatar
minibikini committed
37
         :ok <- validate_limit(user, admin_id),
minibikini's avatar
minibikini committed
38
         {:ok, backup} <- user |> new() |> Repo.insert() do
minibikini's avatar
minibikini committed
39
      BackupWorker.process(backup, admin_id)
minibikini's avatar
minibikini committed
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    end
  end

  def new(user) do
    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"

    %__MODULE__{
      user_id: user.id,
      content_type: "application/zip",
      file_name: name
    }
  end

minibikini's avatar
minibikini committed
55
56
57
  def delete(backup) do
    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])

minibikini's avatar
minibikini committed
58
    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
minibikini's avatar
minibikini committed
59
60
61
62
      Repo.delete(backup)
    end
  end

minibikini's avatar
minibikini committed
63
64
65
  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok

  defp validate_limit(user, nil) do
minibikini's avatar
minibikini committed
66
67
    case get_last(user.id) do
      %__MODULE__{inserted_at: inserted_at} ->
68
        days = Pleroma.Config.get([__MODULE__, :limit_days])
minibikini's avatar
minibikini committed
69
70
71
72
73
        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)

        if diff > days do
          :ok
        else
minibikini's avatar
minibikini committed
74
75
76
77
78
79
80
81
          {:error,
           dngettext(
             "errors",
             "Last export was less than a day ago",
             "Last export was less than %{days} days ago",
             days,
             days: days
           )}
minibikini's avatar
minibikini committed
82
83
84
85
86
87
88
        end

      nil ->
        :ok
    end
  end

minibikini's avatar
minibikini committed
89
90
91
92
  defp validate_email_enabled do
    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
      :ok
    else
minibikini's avatar
minibikini committed
93
      {:error, dgettext("errors", "Backups require enabled email")}
minibikini's avatar
minibikini committed
94
95
96
    end
  end

minibikini's avatar
minibikini committed
97
98
99
100
  defp validate_user_email(%User{email: nil}) do
    {:error, dgettext("errors", "Email is required")}
  end

minibikini's avatar
minibikini committed
101
102
  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok

minibikini's avatar
minibikini committed
103
104
105
106
107
108
109
110
  def get_last(user_id) do
    __MODULE__
    |> where(user_id: ^user_id)
    |> order_by(desc: :id)
    |> limit(1)
    |> Repo.one()
  end

minibikini's avatar
minibikini committed
111
112
113
114
115
116
117
  def list(%User{id: user_id}) do
    __MODULE__
    |> where(user_id: ^user_id)
    |> order_by(desc: :id)
    |> Repo.all()
  end

minibikini's avatar
minibikini committed
118
119
120
121
  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
    __MODULE__
    |> where(user_id: ^user_id)
    |> where([b], b.id != ^latest_id)
minibikini's avatar
minibikini committed
122
123
    |> Repo.all()
    |> Enum.each(&BackupWorker.delete/1)
minibikini's avatar
minibikini committed
124
125
126
127
  end

  def get(id), do: Repo.get(__MODULE__, id)

minibikini's avatar
minibikini committed
128
  def process(%__MODULE__{} = backup) do
minibikini's avatar
minibikini committed
129
    with {:ok, zip_file} <- export(backup),
minibikini's avatar
minibikini committed
130
131
132
133
134
135
136
         {:ok, %{size: size}} <- File.stat(zip_file),
         {:ok, _upload} <- upload(backup, zip_file) do
      backup
      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
      |> Repo.update()
    end
  end
minibikini's avatar
minibikini committed
137

minibikini's avatar
minibikini committed
138
  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
minibikini's avatar
minibikini committed
139
  def export(%__MODULE__{} = backup) do
minibikini's avatar
minibikini committed
140
141
    backup = Repo.preload(backup, :user)
    name = String.trim_trailing(backup.file_name, ".zip")
142
    dir = dir(name)
minibikini's avatar
minibikini committed
143

minibikini's avatar
minibikini committed
144
145
146
147
148
149
150
    with :ok <- File.mkdir(dir),
         :ok <- actor(dir, backup.user),
         :ok <- statuses(dir, backup.user),
         :ok <- likes(dir, backup.user),
         :ok <- bookmarks(dir, backup.user),
         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
         {:ok, _} <- File.rm_rf(dir) do
151
      {:ok, to_string(zip_path)}
minibikini's avatar
minibikini committed
152
153
154
    end
  end

155
156
157
158
159
  def dir(name) do
    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
    Path.join(dir, name)
  end

minibikini's avatar
minibikini committed
160
  def upload(%__MODULE__{} = backup, zip_path) do
minibikini's avatar
minibikini committed
161
162
163
    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])

    upload = %Pleroma.Upload{
minibikini's avatar
minibikini committed
164
      name: backup.file_name,
minibikini's avatar
minibikini committed
165
      tempfile: zip_path,
minibikini's avatar
minibikini committed
166
      content_type: backup.content_type,
minibikini's avatar
minibikini committed
167
      path: Path.join("backups", backup.file_name)
minibikini's avatar
minibikini committed
168
169
    }

minibikini's avatar
minibikini committed
170
171
    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
         :ok <- File.rm(zip_path) do
minibikini's avatar
minibikini committed
172
      {:ok, upload}
minibikini's avatar
minibikini committed
173
174
175
    end
  end

minibikini's avatar
minibikini committed
176
  defp actor(dir, user) do
minibikini's avatar
minibikini committed
177
178
179
180
    with {:ok, json} <-
           UserView.render("user.json", %{user: user})
           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
           |> Jason.encode() do
minibikini's avatar
minibikini committed
181
      File.write(Path.join(dir, "actor.json"), json)
minibikini's avatar
minibikini committed
182
183
184
185
186
187
188
189
190
191
192
193
    end
  end

  defp write_header(file, name) do
    IO.write(
      file,
      """
      {
        "@context": "https://www.w3.org/ns/activitystreams",
        "id": "#{name}.json",
        "type": "OrderedCollection",
        "orderedItems": [
minibikini's avatar
minibikini committed
194

minibikini's avatar
minibikini committed
195
196
197
198
199
      """
    )
  end

  defp write(query, dir, name, fun) do
minibikini's avatar
minibikini committed
200
    path = Path.join(dir, "#{name}.json")
minibikini's avatar
minibikini committed
201
202
203

    with {:ok, file} <- File.open(path, [:write, :utf8]),
         :ok <- write_header(file, name) do
204
205
206
207
      total =
        query
        |> Pleroma.Repo.chunk_stream(100)
        |> Enum.reduce(0, fn i, acc ->
208
209
          with {:ok, data} <- fun.(i),
               {:ok, str} <- Jason.encode(data),
210
211
212
213
214
215
               :ok <- IO.write(file, str <> ",\n") do
            acc + 1
          else
            _ -> acc
          end
        end)
minibikini's avatar
minibikini committed
216
217
218
219
220
221
222

      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
        File.close(file)
      end
    end
  end

minibikini's avatar
minibikini committed
223
  defp bookmarks(dir, %{id: user_id} = _user) do
minibikini's avatar
minibikini committed
224
225
226
227
    Bookmark
    |> where(user_id: ^user_id)
    |> join(:inner, [b], activity in assoc(b, :activity))
    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
228
    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
minibikini's avatar
minibikini committed
229
230
  end

minibikini's avatar
minibikini committed
231
  defp likes(dir, user) do
minibikini's avatar
minibikini committed
232
233
234
235
    user.ap_id
    |> Activity.Queries.by_actor()
    |> Activity.Queries.by_type("Like")
    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
236
    |> write(dir, "likes", fn a -> {:ok, a.object} end)
minibikini's avatar
minibikini committed
237
238
  end

minibikini's avatar
minibikini committed
239
  defp statuses(dir, user) do
minibikini's avatar
minibikini committed
240
241
242
    opts =
      %{}
      |> Map.put(:type, ["Create", "Announce"])
minibikini's avatar
minibikini committed
243
      |> Map.put(:actor_id, user.ap_id)
minibikini's avatar
minibikini committed
244

minibikini's avatar
minibikini committed
245
246
247
248
249
    [
      [Pleroma.Constants.as_public(), user.ap_id],
      User.following(user),
      Pleroma.List.memberships(user)
    ]
minibikini's avatar
minibikini committed
250
251
252
253
    |> Enum.concat()
    |> ActivityPub.fetch_activities_query(opts)
    |> write(dir, "outbox", fn a ->
      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
254
        {:ok, Map.delete(activity, "@context")}
minibikini's avatar
minibikini committed
255
256
257
258
      end
    end)
  end
end