Verified Commit e7ac1590 authored by Alex Gleason's avatar Alex Gleason
Browse files

Merge remote-tracking branch 'upstream/develop' into oauth-token-id

parents b6a69b5e 115673bc
......@@ -8,7 +8,9 @@ variables: &global_variables
MIX_ENV: test
cache: &global_cache_policy
key: ${CI_COMMIT_REF_SLUG}
key:
files:
- mix.lock
paths:
- deps
- _build
......@@ -171,8 +173,8 @@ spec-deploy:
- apk add curl
script:
- curl -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline
stop_review_app:
image: alpine:3.9
stage: deploy
......@@ -231,7 +233,7 @@ amd64-musl:
stage: release
artifacts: *release-artifacts
only: *release-only
image: elixir:1.10.3-alpine
image: elixir:1.10.3-alpine
cache: *release-cache
variables: *release-variables
before_script: &before-release-musl
......
......@@ -6,13 +6,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## Unreleased
### Changed
- The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
### Added
- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
## Unreleased (Patch)
### Fixed
- Try to save exported ConfigDB settings (migrate_from_db) in the system temp directory if default location is not writable.
- Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image
- Applying ConcurrentLimiter settings via AdminAPI
- User login failures if their `notification_settings` were in a NULL state.
- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
## [2.3.0] - 2020-03-01
......
......@@ -409,6 +409,8 @@
threshold: 604_800,
actions: [:delist, :strip_followers]
config :pleroma, :mrf_follow_bot, follower_nickname: nil
config :pleroma, :rich_media,
enabled: true,
ignore_hosts: [],
......
......@@ -2942,6 +2942,23 @@
}
]
},
%{
group: :pleroma,
key: :mrf_follow_bot,
tab: :mrf,
related_policy: "Pleroma.Web.ActivityPub.MRF.FollowBotPolicy",
label: "MRF FollowBot Policy",
type: :group,
description: "Automatically follows newly discovered accounts.",
children: [
%{
key: :follower_nickname,
type: :string,
description: "The name of the bot account to use for following newly discovered users.",
suggestions: ["followbot"]
}
]
},
%{
group: :pleroma,
key: :modules,
......
......@@ -124,6 +124,7 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
* `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
* `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
* `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
* `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
......@@ -220,6 +221,11 @@ Notes:
- The hashtags in the configuration do not have a leading `#`.
- This MRF Policy is always enabled, if you want to disable it you have to set empty lists
#### :mrf_follow_bot
* `follower_nickname`: The name of the bot account to use for following newly discovered users. Using `followbot` or similar is strongly suggested.
### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed
* `outgoing_blocks`: Whether to federate blocks to other instances
......
......@@ -38,6 +38,7 @@ Has these additional fields under the `pleroma` object:
- `thread_muted`: true if the thread the post belongs to is muted
- `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
- `parent_visible`: If the parent of this post is visible to the user or not.
- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
## Scheduled statuses
......
......@@ -20,7 +20,7 @@ The default front-end used by Pleroma is Pleroma-FE. You can find more informati
### Mastodon interface
If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
Just add a "/web" after your instance url (e.g. <https://pleroma.soykaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.
......@@ -290,7 +290,7 @@ nginx -t
## Create your first user and set as admin
```sh
cd /opt/pleroma/bin
cd /opt/pleroma
su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
```
This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
......
......@@ -184,40 +184,48 @@ def get_by_ap_id_with_object(ap_id) do
|> Repo.one()
end
@spec get_by_id(String.t()) :: Activity.t() | nil
def get_by_id(id) do
case FlakeId.flake_id?(id) do
true ->
Activity
|> where([a], a.id == ^id)
|> restrict_deactivated_users()
|> Repo.one()
_ ->
nil
end
end
def get_by_id_with_user_actor(id) do
case FlakeId.flake_id?(id) do
true ->
Activity
|> where([a], a.id == ^id)
|> with_preloaded_user_actor()
|> Repo.one()
_ ->
nil
@doc """
Gets activity by ID, doesn't load activities from deactivated actors by default.
"""
@spec get_by_id(String.t(), keyword()) :: t() | nil
def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts)
@spec get_by_id_with_user_actor(String.t()) :: t() | nil
def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor])
@spec get_by_id_with_object(String.t()) :: t() | nil
def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object])
defp get_by_id_with_opts(id, opts) do
if FlakeId.flake_id?(id) do
query = Queries.by_id(id)
with_filters_query =
if is_list(opts[:filter]) do
Enum.reduce(opts[:filter], query, fn
{:type, type}, acc -> Queries.by_type(acc, type)
:restrict_deactivated, acc -> restrict_deactivated_users(acc)
_, acc -> acc
end)
else
query
end
with_preloads_query =
if is_list(opts[:preload]) do
Enum.reduce(opts[:preload], with_filters_query, fn
:user_actor, acc -> with_preloaded_user_actor(acc)
:object, acc -> with_preloaded_object(acc)
_, acc -> acc
end)
else
with_filters_query
end
Repo.one(with_preloads_query)
end
end
def get_by_id_with_object(id) do
Activity
|> where(id: ^id)
|> with_preloaded_object()
|> Repo.one()
end
def all_by_ids_with_object(ids) do
Activity
|> where([a], a.id in ^ids)
......@@ -269,6 +277,11 @@ def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
def get_create_by_object_ap_id_with_object(_), do: nil
@spec create_by_id_with_object(String.t()) :: t() | nil
def create_by_id_with_object(id) do
get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"])
end
defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
get_create_by_object_ap_id_with_object(ap_id)
end
......@@ -368,12 +381,6 @@ def direct_conversation_id(activity, for_user) do
end
end
@spec pinned_by_actor?(Activity.t()) :: boolean()
def pinned_by_actor?(%Activity{} = activity) do
actor = user_actor(activity)
activity.id in actor.pinned_activities
end
@spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
ap_id
......@@ -384,4 +391,13 @@ def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
end
def get_by_object_ap_id_with_object(_), do: nil
@spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t()
def add_by_params_query(object_id, actor, target) do
object_id
|> Queries.by_object_id()
|> Queries.by_type("Add")
|> Queries.by_actor(actor)
|> where([a], fragment("?->>'target' = ?", a.data, ^target))
end
end
......@@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do
alias Pleroma.Activity
alias Pleroma.User
@spec by_id(query(), String.t()) :: query()
def by_id(query \\ Activity, id) do
from(a in query, where: a.id == ^id)
end
@spec by_ap_id(query, String.t()) :: query
def by_ap_id(query \\ Activity, ap_id) do
from(
......
defmodule Pleroma.Config.ReleaseRuntimeProvider do
@moduledoc """
Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.
"""
@behaviour Config.Provider
......@@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
def init(opts), do: opts
@impl true
def load(config, _opts) do
def load(config, opts) do
with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
config_path =
opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
with_runtime_config =
if File.exists?(config_path) do
......@@ -24,7 +25,7 @@ def load(config, _opts) do
warning = [
IO.ANSI.red(),
IO.ANSI.bright(),
"!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
"!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
IO.ANSI.reset()
]
......@@ -33,13 +34,14 @@ def load(config, _opts) do
end
exported_config_path =
config_path
|> Path.dirname()
|> Path.join("prod.exported_from_db.secret.exs")
opts[:exported_config_path] ||
config_path
|> Path.dirname()
|> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")
with_exported =
if File.exists?(exported_config_path) do
exported_config = Config.Reader.read!(with_runtime_config)
exported_config = Config.Reader.read!(exported_config_path)
Config.Reader.merge(with_runtime_config, exported_config)
else
with_runtime_config
......
......@@ -387,6 +387,6 @@ defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
@spec module_name?(String.t()) :: boolean()
def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger"]
string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
end
end
......@@ -71,6 +71,14 @@ def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(oth
compare_uris(id_uri, other_uri)
end
# Mastodon pin activities don't have an id, so we check the object field, which will be pinned.
def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do
id_uri = URI.parse(id)
object_uri = URI.parse(object)
compare_uris(id_uri, object_uri)
end
def contain_origin_from_id(_id, _data), do: :error
def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
......
......@@ -99,6 +99,7 @@ defmodule Pleroma.User do
field(:local, :boolean, default: true)
field(:follower_address, :string)
field(:following_address, :string)
field(:featured_address, :string)
field(:search_rank, :float, virtual: true)
field(:search_type, :integer, virtual: true)
field(:tags, {:array, :string}, default: [])
......@@ -130,7 +131,6 @@ defmodule Pleroma.User do
field(:hide_followers, :boolean, default: false)
field(:hide_follows, :boolean, default: false)
field(:hide_favorites, :boolean, default: true)
field(:pinned_activities, {:array, :string}, default: [])
field(:email_notifications, :map, default: %{"digest" => false})
field(:mascot, :map, default: nil)
field(:emoji, :map, default: %{})
......@@ -148,6 +148,7 @@ defmodule Pleroma.User do
field(:accepts_chat_messages, :boolean, default: nil)
field(:last_active_at, :naive_datetime)
field(:disclose_client, :boolean, default: true)
field(:pinned_objects, :map, default: %{})
embeds_one(
:notification_settings,
......@@ -372,8 +373,10 @@ def banner_url(user, options \\ []) do
end
# Should probably be renamed or removed
@spec ap_id(User.t()) :: String.t()
def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
@spec ap_followers(User.t()) :: String.t()
def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
......@@ -381,6 +384,11 @@ def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
@spec ap_featured_collection(User.t()) :: String.t()
def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa
def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured"
defp truncate_fields_param(params) do
if Map.has_key?(params, :fields) do
Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
......@@ -443,6 +451,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
:uri,
:follower_address,
:following_address,
:featured_address,
:hide_followers,
:hide_follows,
:hide_followers_count,
......@@ -454,7 +463,8 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
:invisible,
:actor_type,
:also_known_as,
:accepts_chat_messages
:accepts_chat_messages,
:pinned_objects
]
)
|> cast(params, [:name], empty_values: [])
......@@ -686,7 +696,7 @@ def register_changeset_ldap(struct, params = %{password: password})
|> validate_format(:nickname, local_nickname_regex())
|> put_ap_id()
|> unique_constraint(:ap_id)
|> put_following_and_follower_address()
|> put_following_and_follower_and_featured_address()
end
def register_changeset(struct, params \\ %{}, opts \\ []) do
......@@ -747,7 +757,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
|> put_password_hash
|> put_ap_id()
|> unique_constraint(:ap_id)
|> put_following_and_follower_address()
|> put_following_and_follower_and_featured_address()
end
def maybe_validate_required_email(changeset, true), do: changeset
......@@ -765,11 +775,16 @@ defp put_ap_id(changeset) do
put_change(changeset, :ap_id, ap_id)
end
defp put_following_and_follower_address(changeset) do
followers = ap_followers(%User{nickname: get_field(changeset, :nickname)})
defp put_following_and_follower_and_featured_address(changeset) do
user = %User{nickname: get_field(changeset, :nickname)}
followers = ap_followers(user)
following = ap_following(user)
featured = ap_featured_collection(user)
changeset
|> put_change(:follower_address, followers)
|> put_change(:following_address, following)
|> put_change(:featured_address, featured)
end
defp autofollow_users(user) do
......@@ -2343,45 +2358,35 @@ def approval_changeset(user, set_approval: approved?) do
cast(user, %{is_approved: approved?}, [:is_approved])
end
def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do
if id not in user.pinned_activities do
max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
params = %{pinned_activities: user.pinned_activities ++ [id]}
# if pinned activity was scheduled for deletion, we remove job
if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do
Oban.cancel_job(expiration.id)
end
@spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()}
def add_pinned_object_id(%User{} = user, object_id) do
if !user.pinned_objects[object_id] do
params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())}
user
|> cast(params, [:pinned_activities])
|> validate_length(:pinned_activities,
max: max_pinned_statuses,
message: "You have already pinned the maximum number of statuses"
)
|> cast(params, [:pinned_objects])
|> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects ->
max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
if Enum.count(pinned_objects) <= max_pinned_statuses do
[]
else
[pinned_objects: "You have already pinned the maximum number of statuses"]
end
end)
else
change(user)
end
|> update_and_set_cache()
end
def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do
params = %{pinned_activities: List.delete(user.pinned_activities, id)}
# if pinned activity was scheduled for deletion, we reschedule it for deletion
if data["expires_at"] do
# MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
{:ok, expires_at} =
data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: id,
expires_at: expires_at
})
end
@spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()}
def remove_pinned_object_id(%User{} = user, object_id) do
user
|> cast(params, [:pinned_activities])
|> cast(
%{pinned_objects: Map.delete(user.pinned_objects, object_id)},
[:pinned_objects]
)
|> update_and_set_cache()
end
......
......@@ -11,6 +11,8 @@ defmodule Pleroma.Utils do
eperm epipe erange erofs espipe esrch estale etxtbsy exdev
)a
@repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000)
def compile_dir(dir) when is_binary(dir) do
dir
|> File.ls!()
......@@ -63,4 +65,21 @@ def posix_error_message(code) when code in @posix_error_codes do
end
def posix_error_message(_), do: ""
@doc """
Returns [timeout: integer] suitable for passing as an option to Repo functions.
This function detects if the execution was triggered from IEx shell, Mix task, or
./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value.
"""
@spec query_timeout() :: [timeout: integer]
def query_timeout do
{parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2)
cond do
parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity]
parent == :erl_eval -> [timeout: :infinity]
true -> [timeout: @repo_timeout]
end
end
end
......@@ -630,7 +630,7 @@ defp fetch_activities_for_user(user, reading_user, params) do
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:user, reading_user)
|> Map.put(:actor_id, user.ap_id)
|> Map.put(:pinned_activity_ids, user.pinned_activities)
|> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))
params =
if User.blocks?(reading_user, user) do
......@@ -1075,8 +1075,18 @@ defp restrict_unlisted(query, %{restrict_unlisted: true}) do
defp restrict_unlisted(query, _), do: query
defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do
from(activity in query, where: activity.id in ^ids)
defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do
from(
[activity, object: o] in query,
where:
fragment(
"(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
activity.data,
activity.data,
activity.data,
^ids
)
)
end
defp restrict_pinned(query, _), do: query
......@@ -1419,6 +1429,9 @@ defp object_to_user_data(data) do
invisible = data["invisible"] || false
actor_type = data["type"] || "Person"
featured_address = data["featured"]
{:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address)
public_key =
if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
data["publicKey"]["publicKeyPem"]
......@@ -1447,13 +1460,15 @@ defp object_to_user_data(data) do
name: data["name"],
follower_address: data["followers"],
following_address: data["following"],
featured_address: featured_address,
bio: data["summary"] || "",
actor_type: actor_type,
also_known_as: Map.get(data, "alsoKnownAs", []),
public_key: public_key,
inbox: data["inbox"],
shared_inbox: shared_inbox,
accepts_chat_messages: accepts_chat_messages
accepts_chat_messages: accepts_chat_messages,
pinned_objects: pinned_objects
}
# nickname can be nil because of virtual actors
......@@ -1591,6 +1606,41 @@ def maybe_handle_clashing_nickname(data) do
end
end
def pin_data_from_featured_collection(%{
"type" => type,
"orderedItems" => objects
})
when type in ["OrderedCollection", "Collection"] do
Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
end
def fetch_and_prepare_featured_from_ap_id(nil) do
{:ok, %{}}
end