Verified Commit 5df42e1c authored by Sergey Suprunenko's avatar Sergey Suprunenko

Merge branch 'develop' into merge/develop-20190910

parents 39de6850 69800055
......@@ -8,19 +8,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- OStatus: eliminate the possibility of a protocol downgrade attack.
- OStatus: prevent following locked accounts, bypassing the approval process.
### Removed
- **Breaking:** GNU Social API with Qvitter extensions support
- **Breaking:** ActivityPub: The `accept_blocks` configuration setting.
- Emoji: Remove longfox emojis.
- Remove `Reply-To` header from report emails for admins.
### Changed
- **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config
- **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired
- **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities.
- Configuration: OpenGraph and TwitterCard providers enabled by default
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
- Mastodon API: `pleroma.thread_muted` key in the Status entity
- Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set
- NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option
- NodeInfo: Return `mailerEnabled` in `metadata`
- Mastodon API: Unsubscribe followers when they unfollow a user
- AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses)
- Improve digest email template
– Pagination: (optional) return `total` alongside with `items` when paginating
### Fixed
- Following from Osada
- Not being able to pin unlisted posts
- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised.
- Favorites timeline doing database-intensive queries
......@@ -28,7 +38,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- `federation_incoming_replies_max_depth` option being ignored in certain cases
- Federation/MediaProxy not working with instances that have wrong certificate order
- Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`)
- Mastodon API: Misskey's endless polls being unable to render
- Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
- Mastodon API: Notifications endpoint crashing if one notification failed to render
- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted
- Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
......@@ -48,6 +60,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances
- MRF: fix use of unserializable keyword lists in describe() implementations
- ActivityPub: Deactivated user deletion
- ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user
- MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled
### Added
- Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically.
......@@ -95,6 +109,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix Tasks: `mix pleroma.database fix_likes_collections`
- Federation: Remove `likes` from objects.
- Admin API: Added moderation log
- Web response cache (currently, enabled for ActivityPub)
- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`)
### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
......@@ -102,10 +118,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- RichMedia: parsers and their order are configured in `rich_media` config.
- RichMedia: add the rich media ttl based on image expiration time.
### Removed
- Emoji: Remove longfox emojis.
- Remove `Reply-To` header from report emails for admins.
- ActivityPub: The `accept_blocks` configuration setting.
## [1.0.1] - 2019-07-14
### Security
......
FROM rinpatch/elixir:1.9.0-rc.0-alpine as build
FROM elixir:1.9-alpine as build
COPY . .
......@@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make &&\
mkdir release &&\
mix release --path release
FROM alpine:latest
FROM alpine:3.9
ARG HOME=/opt/pleroma
ARG DATA=/var/lib/pleroma
......
......@@ -8,7 +8,7 @@ Pleroma is a microblogging server software that can federate (= exchange message
Pleroma is written in Elixir, high-performance and can run on small devices like a Raspberry Pi.
For clients it supports both the [GNU Social API with Qvitter extensions](https://twitter-api.readthedocs.io/en/latest/index.html) and the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/).
For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/) with Pleroma extensions (see "Pleroma's APIs and Mastodon API extensions" section on <https://docs-develop.pleroma.social>).
- [Client Applications for Pleroma](https://docs-develop.pleroma.social/clients.html)
......
......@@ -602,6 +602,10 @@ config :pleroma, :rate_limit, nil
config :pleroma, Pleroma.ActivityExpiration, enabled: true
config :pleroma, :web_cache_ttl,
activity_pub: nil,
activity_pub_question: 30_000
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
......@@ -10,7 +10,7 @@ config :pleroma, :instance,
notify_email: System.get_env("NOTIFY_EMAIL"),
limit: 5000,
registrations_open: false,
dynamic_configuration: true
healthcheck: true
config :pleroma, Pleroma.Repo,
adapter: Ecto.Adapters.Postgres,
......
......@@ -26,6 +26,7 @@ Has these additional fields under the `pleroma` object:
- `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
- `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
- `thread_muted`: true if the thread the post belongs to is muted
## Attachments
......@@ -90,6 +91,20 @@ Additional parameters can be added to the JSON body/Form data:
- `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
- `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
## GET `/api/v1/statuses`
An endpoint to get multiple statuses by IDs.
Required parameters:
- `ids`: array of activity ids
Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`.
Returns: array of Status.
The maximum number of statuses is limited to 100 per request.
## PATCH `/api/v1/update_credentials`
Additional parameters can be added to the JSON body/Form data:
......
......@@ -126,13 +126,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
## `/api/pleroma/admin/`…
See [Admin-API](Admin-API.md)
## `/api/pleroma/notifications/read`
### Mark a single notification as read
## `/api/v1/pleroma/notifications/read`
### Mark notifications as read
* Method `POST`
* Authentication: required
* Params:
* `id`: notification's id
* Response: JSON. Returns `{"status": "success"}` if the reading was successful, otherwise returns `{"error": "error_msg"}`
* Params (mutually exclusive):
* `id`: a single notification id to read
* `max_id`: read all notifications up to this id
* Response: Notification entity/Array of Notification entities that were read. In case of `max_id`, only the first 80 read notifications will be returned.
## `/api/v1/pleroma/accounts/:id/subscribe`
### Subscribe to receive notifications for all statuses posted by a user
......
......@@ -690,3 +690,12 @@ Supported rate limiters:
* `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
## :web_cache_ttl
The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.
Available caches:
* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration).
* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds).
......@@ -71,26 +71,26 @@ server {
proxy_set_header Connection "upgrade";
proxy_set_header Host $http_host;
# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
# and `localhost.` resolves to [::0] on some systems: see issue #930
# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
# and `localhost.` resolves to [::0] on some systems: see issue #930
proxy_pass http://127.0.0.1:4000;
client_max_body_size 16m;
}
location ~ ^/(media|proxy) {
proxy_cache pleroma_media_cache;
proxy_cache pleroma_media_cache;
slice 1m;
proxy_cache_key $host$uri$is_args$args$slice_range;
proxy_set_header Range $slice_range;
proxy_http_version 1.1;
proxy_cache_valid 200 206 301 304 1h;
proxy_cache_lock on;
proxy_cache_lock on;
proxy_ignore_client_abort on;
proxy_buffering on;
proxy_buffering on;
chunked_transfer_encoding on;
proxy_ignore_headers Cache-Control;
proxy_hide_header Cache-Control;
proxy_pass http://localhost:4000;
proxy_hide_header Cache-Control;
proxy_pass http://127.0.0.1:4000;
}
}
......@@ -173,6 +173,13 @@ defmodule Pleroma.Activity do
|> Repo.one()
end
def all_by_ids_with_object(ids) do
Activity
|> where([a], a.id in ^ids)
|> with_preloaded_object()
|> Repo.all()
end
def by_object_ap_id(ap_id) do
from(
activity in Activity,
......@@ -308,10 +315,19 @@ defmodule Pleroma.Activity do
%{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
_ -> nil
end)
|> purge_web_resp_cache()
end
def delete_by_ap_id(_), do: nil
defp purge_web_resp_cache(%Activity{} = activity) do
%{path: path} = URI.parse(activity.data["id"])
Cachex.del(:web_resp_cache, path)
activity
end
defp purge_web_resp_cache(nil), do: nil
for {ap_type, type} <- @mastodon_notification_types do
def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
do: unquote(type)
......@@ -362,12 +378,12 @@ defmodule Pleroma.Activity do
end
def restrict_deactivated_users(query) do
deactivated_users =
from(u in User.Query.build(deactivated: true), select: u.ap_id)
|> Repo.all()
from(activity in query,
where:
fragment(
"? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
activity.actor
)
where: activity.actor not in ^deactivated_users
)
end
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Activity.Queries do
@moduledoc """
Contains queries for Activity.
"""
import Ecto.Query, only: [from: 2]
@type query :: Ecto.Queryable.t() | Activity.t()
alias Pleroma.Activity
@spec by_actor(query, String.t()) :: query
def by_actor(query \\ Activity, actor) do
from(
activity in query,
where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
)
end
@spec by_object_id(query, String.t()) :: query
def by_object_id(query \\ Activity, object_id) do
from(activity in query,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
activity.data,
^object_id
)
)
end
@spec by_type(query, String.t()) :: query
def by_type(query \\ Activity, activity_type) do
from(
activity in query,
where: fragment("(?)->>'type' = ?", activity.data, ^activity_type)
)
end
@spec limit(query, pos_integer()) :: query
def limit(query \\ Activity, limit) do
from(activity in query, limit: ^limit)
end
end
......@@ -117,7 +117,8 @@ defmodule Pleroma.Application do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500)
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500)
]
end
......
......@@ -109,15 +109,19 @@ defmodule Pleroma.List do
end
def create(title, %User{} = creator) do
list = %Pleroma.List{user_id: creator.id, title: title}
Repo.transaction(fn ->
list = Repo.insert!(list)
list
|> change(ap_id: "#{creator.ap_id}/lists/#{list.id}")
|> Repo.update!()
end)
changeset = title_changeset(%Pleroma.List{user_id: creator.id}, %{title: title})
if changeset.valid? do
Repo.transaction(fn ->
list = Repo.insert!(changeset)
list
|> change(ap_id: "#{creator.ap_id}/lists/#{list.id}")
|> Repo.update!()
end)
else
{:error, changeset}
end
end
def follow(%Pleroma.List{following: following} = list, %User{} = followed) do
......
......@@ -102,15 +102,33 @@ defmodule Pleroma.Notification do
n in Notification,
where: n.user_id == ^user_id,
where: n.id <= ^id,
where: n.seen == false,
update: [
set: [
seen: true,
updated_at: ^NaiveDateTime.utc_now()
]
]
],
# Ideally we would preload object and activities here
# but Ecto does not support preloads in update_all
select: n.id
)
Repo.update_all(query, [])
{_, notification_ids} = Repo.update_all(query, [])
Notification
|> where([n], n.id in ^notification_ids)
|> join(:inner, [n], activity in assoc(n, :activity))
|> join(:left, [n, a], object in Object,
on:
fragment(
"(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
object.data,
a.data
)
)
|> preload([n, a, o], activity: {a, object: o})
|> Repo.all()
end
def read_one(%User{} = user, notification_id) do
......
......@@ -132,14 +132,16 @@ defmodule Pleroma.Object do
def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_by_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object, deleted_activity}
end
end
def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object}
end
end
......@@ -152,8 +154,6 @@ defmodule Pleroma.Object do
def update_and_set_cache(changeset) do
with {:ok, object} <- Repo.update(changeset) do
set_cache(object)
else
e -> e
end
end
......
......@@ -16,6 +16,15 @@ defmodule Pleroma.Pagination do
def fetch_paginated(query, params, type \\ :keyset)
def fetch_paginated(query, %{"total" => true} = params, :keyset) do
total = Repo.aggregate(query, :count, :id)
%{
total: total,
items: fetch_paginated(query, Map.drop(params, ["total"]), :keyset)
}
end
def fetch_paginated(query, params, :keyset) do
options = cast_params(params)
......@@ -25,6 +34,15 @@ defmodule Pleroma.Pagination do
|> enforce_order(options)
end
def fetch_paginated(query, %{"total" => true} = params, :offset) do
total = Repo.aggregate(query, :count, :id)
%{
total: total,
items: fetch_paginated(query, Map.drop(params, ["total"]), :offset)
}
end
def fetch_paginated(query, params, :offset) do
options = cast_params(params)
......
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.Cache do
@moduledoc """
Caches successful GET responses.
To enable the cache add the plug to a router pipeline or controller:
plug(Pleroma.Plugs.Cache)
## Configuration
To configure the plug you need to pass settings as the second argument to the `plug/2` macro:
plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true])
Available options:
- `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`.
- `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`.
Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct:
def index(conn, _params) do
ttl = 60_000 # one minute
conn
|> assign(:cache_ttl, ttl)
|> render("index.html")
end
"""
import Phoenix.Controller, only: [current_path: 1, json: 2]
import Plug.Conn
@behaviour Plug
@defaults %{ttl: nil, query_params: true}
@impl true
def init([]), do: @defaults
def init(opts) do
opts = Map.new(opts)
Map.merge(@defaults, opts)
end
@impl true
def call(%{method: "GET"} = conn, opts) do
key = cache_key(conn, opts)
case Cachex.get(:web_resp_cache, key) do
{:ok, nil} ->
cache_resp(conn, opts)
{:ok, record} ->
send_cached(conn, record)
{atom, message} when atom in [:ignore, :error] ->
render_error(conn, message)
end
end
def call(conn, _), do: conn
# full path including query params
defp cache_key(conn, %{query_params: true}), do: current_path(conn)
# request path without query params
defp cache_key(conn, %{query_params: false}), do: conn.request_path
# request path with specific query params
defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do
query_string =
conn.params
|> Map.take(query_params)
|> URI.encode_query()
conn.request_path <> "?" <> query_string
end
defp cache_resp(conn, opts) do
register_before_send(conn, fn
%{status: 200, resp_body: body} = conn ->
ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl)
key = cache_key(conn, opts)
content_type = content_type(conn)
record = {content_type, body}
Cachex.put(:web_resp_cache, key, record, ttl: ttl)
put_resp_header(conn, "x-cache", "MISS from Pleroma")
conn ->
conn
end)
end
defp content_type(conn) do
conn
|> Plug.Conn.get_resp_header("content-type")
|> hd()
end
defp send_cached(conn, {content_type, body}) do
conn
|> put_resp_content_type(content_type, nil)
|> put_resp_header("x-cache", "HIT from Pleroma")
|> send_resp(:ok, body)
|> halt()
end
defp render_error(conn, message) do
conn
|> put_status(:internal_server_error)
|> json(%{error: message})
|> halt()
end
end
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.TrailingFormatPlug do
@moduledoc "Calls TrailingFormatPlug for specific paths. Ideally we would just do this in the router, but TrailingFormatPlug needs to be called before Plug.Parsers."
@behaviour Plug
@paths [
"/api/statusnet",
"/api/statuses",
"/api/qvitter",
"/api/search",
"/api/account",
"/api/friends",
"/api/mutes",
"/api/media",
"/api/favorites",
"/api/blocks",
"/api/friendships",
"/api/users",
"/users",
"/nodeinfo",
"/api/help",
"/api/externalprofile",
"/notice",
"/api/pleroma/emoji"
]
def init(opts) do
TrailingFormatPlug.init(opts)
end
for path <- @paths do
def call(%{request_path: unquote(path) <> _} = conn, opts) do
TrailingFormatPlug.call(conn, opts)
end
end
def call(conn, _opts), do: conn
end
......@@ -597,8 +597,22 @@ defmodule Pleroma.User do
end)
end
def get_cached_by_nickname_or_id(nickname_or_id) do
get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
def get_cached_by_nickname_or_id(nickname_or_id, opts \\ []) do
restrict_to_local = Pleroma.Config.get([:instance, :limit_to_local_content])
cond do
is_integer(nickname_or_id) or Pleroma.FlakeId.is_flake_id?(nickname_or_id) ->
get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
restrict_to_local == false ->
get_cached_by_nickname(nickname_or_id)
restrict_to_local == :unauthenticated and match?(%User{}, opts[:for]) ->
get_cached_by_nickname(nickname_or_id)
true ->
nil
end
end
def get_by_nickname(nickname) do
......
......@@ -139,7 +139,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
# Splice in the child object if we have one.
activity =
if !is_nil(object) do
if not is_nil(object) do
Map.put(activity, :object, object)
else
activity
......@@ -331,12 +331,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
end
def unlike(
%User{} = actor,
%Object{} = object,
activity_id \\ nil,
local \\ true
) do
def unlike(%User{} = actor, %Object{} = object, activity_id \\ nil, local \\ true) do
with %Activity{} = like_activity <- get_existing_like(actor.ap_id, object),
unlike_data <- make_unlike_data(actor, like_activity, activity_id),
{:ok, unlike_activity} <- insert(unlike_data, local),
......@@ -813,7 +808,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
)
unless opts["skip_preload"] do
from([thread_mute: tm] in query, where: is_nil(tm))
from([thread_mute: tm] in query, where: is_nil(tm.user_id))
else
query
end
......
......@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
action_fallback(:errors)
plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object])
plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
......@@ -53,8 +54,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
%Object{} = object <- Object.get_cached_by_ap_id(ap_id),
{_, true} <- {:public?, Visibility.is_public?(object)} do
conn
|> set_cache_ttl_for(object)
|> put_resp_content_type("application/activity+json")
|> json(ObjectView.render("object.json", %{object: object}))
|> put_view(ObjectView)
|> render("object.json", object: object)
else
{:public?, false} ->
{:error, :not_found