Merge branch 'develop' into 'fix/2411-mutes-api'

# Conflicts:
#   CHANGELOG.md
#   docs/development/API/differences_in_mastoapi_responses.md

CHANGELOG.md

@@ -49,6 +49,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: An endpoint to manage frontends.
 - Streaming API: Add follow relationships updates.
 - WebPush: Introduce `pleroma:chat_mention` and `pleroma:emoji_reaction` notification types.
+- Mastodon API: Add monthly active users to `/api/v1/instance` (`pleroma.stats.mau`).
+- Mastodon API: Home, public, hashtag & list timelines accept `only_media`, `remote` & `local` parameters for filtration.
 - Mastodon API: `/api/v1/accounts/:id` & `/api/v1/mutes` endpoints accept `with_relationships` parameter and return filled `pleroma.relationship` field.
 </details>
 
@@ -59,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Creating incorrect IPv4 address-style HTTP links when encountering certain numbers.
 - Reblog API Endpoint: Do not set visibility parameter to public by default and let CommonAPI to infer it from status, so a user can reblog their private status without explicitly setting reblog visibility to private.
 - Tag URLs in statuses are now absolute
+- Removed duplicate jobs to purge expired activities
 
 <details>
   <summary>API Changes</summary>

config/description.exs

@@ -3224,6 +3224,12 @@
         type: :string,
         description: "S3 host",
         suggestions: ["s3.eu-central-1.amazonaws.com"]
+      },
+      %{
+        key: :region,
+        type: :string,
+        description: "S3 region (for AWS)",
+        suggestions: ["us-east-1"]
       }
     ]
   },

docs/development/API/differences_in_mastoapi_responses.md

@@ -16,6 +16,12 @@ Adding the parameter `reply_visibility` to the public and home timelines queries
 
 Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
 
+Home, public, hashtag & list timelines accept these parameters:
+
+- `only_media`: show only statuses with media attached
+- `local`: show only local statuses
+- `remote`: show only remote statuses
+
 ## Statuses
 
 - `visibility`: has additional possible values `list` and `local` (for local-only statuses)
@@ -54,6 +60,16 @@ The `id` parameter can also be the `nickname` of the user. This only works in th
 - `/api/v1/accounts/:id`
 - `/api/v1/accounts/:id/statuses`
 
+`/api/v1/accounts/:id/statuses` endpoint accepts these parameters:
+
+- `pinned`: include only pinned statuses
+- `tagged`: with tag
+- `only_media`: include only statuses with media attached
+- `with_muted`: include statuses/reactions from muted accounts
+- `exclude_reblogs`: exclude reblogs
+- `exclude_replies`: exclude replies
+- `exclude_visibilities`: exclude visibilities
+
 Endpoints which accept `with_relationships` parameter:
 
 - `/api/v1/accounts/:id`

lib/pleroma/user.ex

@@ -146,6 +146,7 @@ defmodule Pleroma.User do
     field(:inbox, :string)
     field(:shared_inbox, :string)
     field(:accepts_chat_messages, :boolean, default: nil)
+    field(:last_active_at, :naive_datetime)
 
     embeds_one(
       :notification_settings,
@@ -2444,4 +2445,19 @@ def sanitize_html(%User{} = user, filter) do
   def get_host(%User{ap_id: ap_id} = _user) do
     URI.parse(ap_id).host
   end
+
+  def update_last_active_at(%__MODULE__{local: true} = user) do
+    user
+    |> cast(%{last_active_at: NaiveDateTime.utc_now()}, [:last_active_at])
+    |> update_and_set_cache()
+  end
+
+  def active_user_count(weeks \\ 4) do
+    active_after = Timex.shift(NaiveDateTime.utc_now(), weeks: -weeks)
+
+    __MODULE__
+    |> where([u], u.last_active_at >= ^active_after)
+    |> where([u], u.local == true)
+    |> Repo.aggregate(:count)
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -735,6 +735,12 @@ defp restrict_local(query, %{local_only: true}) do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_remote(query, %{remote: true}) do
+    from(activity in query, where: activity.local == false)
+  end
+
+  defp restrict_remote(query, _), do: query
+
   defp restrict_actor(query, %{actor_id: actor_id}) do
     from(activity in query, where: activity.actor == ^actor_id)
   end
@@ -1111,6 +1117,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_tag_all(opts)
     |> restrict_since(opts)
     |> restrict_local(opts)
+    |> restrict_remote(opts)
     |> restrict_actor(opts)
     |> restrict_type(opts)
     |> restrict_state(opts)

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -133,7 +133,7 @@ def statuses_operation do
             :with_muted,
             :query,
             BooleanLike,
-            "Include statuses from muted acccounts."
+            "Include statuses from muted accounts."
           ),
           Operation.parameter(:exclude_reblogs, :query, BooleanLike, "Exclude reblogs"),
           Operation.parameter(:exclude_replies, :query, BooleanLike, "Exclude replies"),
@@ -147,7 +147,7 @@ def statuses_operation do
             :with_muted,
             :query,
             BooleanLike,
-            "Include reactions from muted acccounts."
+            "Include reactions from muted accounts."
           )
         ] ++ pagination_params(),
       responses: %{

lib/pleroma/web/api_spec/operations/timeline_operation.ex

@@ -25,6 +25,8 @@ def home_operation do
       security: [%{"oAuth" => ["read:statuses"]}],
       parameters: [
         local_param(),
+        remote_param(),
+        only_media_param(),
         with_muted_param(),
         exclude_visibilities_param(),
         reply_visibility_param() | pagination_params()
@@ -61,6 +63,7 @@ def public_operation do
         local_param(),
         instance_param(),
         only_media_param(),
+        remote_param(),
         with_muted_param(),
         exclude_visibilities_param(),
         reply_visibility_param() | pagination_params()
@@ -107,6 +110,7 @@ def hashtag_operation do
         ),
         local_param(),
         only_media_param(),
+        remote_param(),
         with_muted_param(),
         exclude_visibilities_param() | pagination_params()
       ],
@@ -132,6 +136,9 @@ def list_operation do
           required: true
         ),
         with_muted_param(),
+        local_param(),
+        remote_param(),
+        only_media_param(),
         exclude_visibilities_param() | pagination_params()
       ],
       operationId: "TimelineController.list",
@@ -198,4 +205,13 @@ defp only_media_param do
       "Show only statuses with media attached?"
     )
   end
+
+  defp remote_param do
+    Operation.parameter(
+      :remote,
+      :query,
+      %Schema{allOf: [BooleanLike], default: false},
+      "Show only remote statuses?"
+    )
+  end
 end

lib/pleroma/web/endpoint.ex

@@ -23,6 +23,18 @@ defmodule Pleroma.Web.Endpoint do
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
   # Cache-control headers are duplicated in case we turn off etags in the future
+  plug(
+    Pleroma.Web.Plugs.InstanceStatic,
+    at: "/",
+    from: :pleroma,
+    only: ["emoji", "images"],
+    gzip: true,
+    cache_control_for_etags: "public, max-age=1209600",
+    headers: %{
+      "cache-control" => "public, max-age=1209600"
+    }
+  )
+
   plug(Pleroma.Web.Plugs.InstanceStatic,
     at: "/",
     gzip: true,

lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex

@@ -51,6 +51,8 @@ def home(%{assigns: %{user: user}} = conn, params) do
       |> Map.put(:reply_filtering_user, user)
       |> Map.put(:announce_filtering_user, user)
       |> Map.put(:user, user)
+      |> Map.put(:local_only, params[:local])
+      |> Map.delete(:local)
 
     activities =
       [user.ap_id | User.following(user)]
@@ -190,6 +192,7 @@ def list(%{assigns: %{user: user}} = conn, %{list_id: id} = params) do
         |> Map.put(:blocking_user, user)
         |> Map.put(:user, user)
         |> Map.put(:muting_user, user)
+        |> Map.put(:local_only, params[:local])
 
       # we must filter the following list for the user to avoid leaking statuses the user
       # does not actually have permission to see (for more info, peruse security issue #270).

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -45,6 +45,7 @@ def render("show.json", _) do
           fields_limits: fields_limits(),
           post_formats: Config.get([:instance, :allowed_post_formats])
         },
+        stats: %{mau: Pleroma.User.active_user_count()},
         vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
       }
     }

lib/pleroma/web/plugs/user_tracking_plug.ex

+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.UserTrackingPlug do
+  alias Pleroma.User
+
+  import Plug.Conn, only: [assign: 3]
+
+  @update_interval :timer.hours(24)
+
+  def init(opts), do: opts
+
+  def call(%{assigns: %{user: %User{id: id} = user}} = conn, _) when not is_nil(id) do
+    with true <- needs_update?(user),
+         {:ok, user} <- User.update_last_active_at(user) do
+      assign(conn, :user, user)
+    else
+      _ -> conn
+    end
+  end
+
+  def call(conn, _), do: conn
+
+  defp needs_update?(%User{last_active_at: nil}), do: true
+
+  defp needs_update?(%User{last_active_at: last_active_at}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), last_active_at, :millisecond) >= @update_interval
+  end
+end

lib/pleroma/web/router.ex

@@ -56,6 +56,7 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Web.Plugs.UserEnabledPlug)
     plug(Pleroma.Web.Plugs.SetUserSessionIdPlug)
     plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
+    plug(Pleroma.Web.Plugs.UserTrackingPlug)
   end
 
   pipeline :base_api do

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -17,12 +17,14 @@ def perform(%Job{
           "object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}
         }
       }) do
-    attachments
-    |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
-    |> fetch_objects
-    |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
-    |> filter_objects
-    |> do_clean
+    if Pleroma.Config.get([:instance, :cleanup_attachments], false) do
+      attachments
+      |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
+      |> fetch_objects
+      |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
+      |> filter_objects
+      |> do_clean
+    end
 
     {:ok, :success}
   end

lib/pleroma/workers/purge_expired_activity.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   Worker which purges expired activity.
   """
 
-  use Oban.Worker, queue: :activity_expiration, max_attempts: 1
+  use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity]
 
   import Ecto.Query
 

priv/repo/migrations/20210122151424_add_last_active_at_to_users.exs

+defmodule Pleroma.Repo.Migrations.AddLastActiveAtToUsers do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add(:last_active_at, :naive_datetime)
+    end
+
+    create_if_not_exists(index(:users, [:last_active_at]))
+  end
+end

priv/repo/migrations/20210128092834_remove_duplicates_from_activity_expiration_queue.exs

+defmodule Pleroma.Repo.Migrations.RemoveDuplicatesFromActivityExpirationQueue do
+  use Ecto.Migration
+
+  import Ecto.Query, only: [from: 2]
+
+  def up do
+    duplicate_ids =
+      from(j in Oban.Job,
+        where: j.queue == "activity_expiration",
+        where: j.worker == "Pleroma.Workers.PurgeExpiredActivity",
+        where: j.state == "scheduled",
+        select:
+          {fragment("(?)->>'activity_id'", j.args), fragment("array_agg(?)", j.id), count(j.id)},
+        group_by: fragment("(?)->>'activity_id'", j.args),
+        having: count(j.id) > 1
+      )
+      |> Pleroma.Repo.all()
+      |> Enum.map(fn {_, ids, _} ->
+        max_id = Enum.max(ids)
+        List.delete(ids, max_id)
+      end)
+      |> List.flatten()
+
+    from(j in Oban.Job, where: j.id in ^duplicate_ids)
+    |> Pleroma.Repo.delete_all()
+  end
+
+  def down, do: :noop
+end

priv/static/index.html

-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.9a4c5ede37b2f0230836.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.54838a79dee084ec3dad.js></script><script type=text/javascript src=/static/js/app.eb8f7164fc75862a251d.js></script></body></html>
\ No newline at end of file
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.9a4c5ede37b2f0230836.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.3b02e2e5bd8cdca42216.js></script><script type=text/javascript src=/static/js/app.c6b8a1c86149ed63e6ff.js></script></body></html>
\ No newline at end of file