Commit bb73f100 authored by rinpatch's avatar rinpatch

Merge branch 'release/1.1.1' into 'stable'

1.1.1 Release

See merge request !1857
parents c00797d0 c00b55bc
Pipeline #18795 canceled with stages
in 15 seconds
...@@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. ...@@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [1.1.1] - 2019-10-18
### Fixed
- One of the migrations between 1.0.0 and 1.1.0 wiping user info of the relay user because of unexpected behavior of postgresql's `jsonb_set`, resulting in inability to post in the default configuration. If you were affected, please run the following query in postgres console, the relay user will be recreated automatically:
```
delete from users where ap_id = 'https://your.instance.hostname/relay';
```
- Bad user search matches
## [1.1.0] - 2019-10-14 ## [1.1.0] - 2019-10-14
**Breaking:** The stable branch has been changed from `master` to `stable`. If you want to keep using 1.0, the `release/1.0` branch will receive security updates for 6 months after 1.1 release. **Breaking:** The stable branch has been changed from `master` to `stable`. If you want to keep using 1.0, the `release/1.0` branch will receive security updates for 6 months after 1.1 release.
......
...@@ -54,7 +54,7 @@ def run(["remove_embedded_objects" | args]) do ...@@ -54,7 +54,7 @@ def run(["remove_embedded_objects" | args]) do
Logger.info("Removing embedded objects") Logger.info("Removing embedded objects")
Repo.query!( Repo.query!(
"update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
[], [],
timeout: :infinity timeout: :infinity
) )
...@@ -152,7 +152,7 @@ def run(["fix_likes_collections"]) do ...@@ -152,7 +152,7 @@ def run(["fix_likes_collections"]) do
set: [ set: [
data: data:
fragment( fragment(
"jsonb_set(?, '{likes}', '[]'::jsonb, true)", "safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)",
object.data object.data
) )
] ]
......
...@@ -181,7 +181,7 @@ def increase_replies_count(ap_id) do ...@@ -181,7 +181,7 @@ def increase_replies_count(ap_id) do
data: data:
fragment( fragment(
""" """
jsonb_set(?, '{repliesCount}', safe_jsonb_set(?, '{repliesCount}',
(coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true) (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
""", """,
o.data, o.data,
...@@ -204,7 +204,7 @@ def decrease_replies_count(ap_id) do ...@@ -204,7 +204,7 @@ def decrease_replies_count(ap_id) do
data: data:
fragment( fragment(
""" """
jsonb_set(?, '{repliesCount}', safe_jsonb_set(?, '{repliesCount}',
(greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true) (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
""", """,
o.data, o.data,
......
...@@ -718,7 +718,7 @@ def increase_note_count(%User{} = user) do ...@@ -718,7 +718,7 @@ def increase_note_count(%User{} = user) do
set: [ set: [
info: info:
fragment( fragment(
"jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)", "safe_jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)",
u.info, u.info,
u.info u.info
) )
...@@ -739,7 +739,7 @@ def decrease_note_count(%User{} = user) do ...@@ -739,7 +739,7 @@ def decrease_note_count(%User{} = user) do
set: [ set: [
info: info:
fragment( fragment(
"jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)", "safe_jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)",
u.info, u.info,
u.info u.info
) )
...@@ -812,7 +812,7 @@ def update_follower_count(%User{} = user) do ...@@ -812,7 +812,7 @@ def update_follower_count(%User{} = user) do
set: [ set: [
info: info:
fragment( fragment(
"jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)", "safe_jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
u.info, u.info,
s.count s.count
) )
......
...@@ -4,11 +4,9 @@ ...@@ -4,11 +4,9 @@
defmodule Pleroma.User.Search do defmodule Pleroma.User.Search do
alias Pleroma.Pagination alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
import Ecto.Query import Ecto.Query
@similarity_threshold 0.25
@limit 20 @limit 20
def search(query_string, opts \\ []) do def search(query_string, opts \\ []) do
...@@ -23,18 +21,10 @@ def search(query_string, opts \\ []) do ...@@ -23,18 +21,10 @@ def search(query_string, opts \\ []) do
maybe_resolve(resolve, for_user, query_string) maybe_resolve(resolve, for_user, query_string)
{:ok, results} = results =
Repo.transaction(fn -> query_string
Ecto.Adapters.SQL.query( |> search_query(for_user, following)
Repo, |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
"select set_limit(#{@similarity_threshold})",
[]
)
query_string
|> search_query(for_user, following)
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
end)
results results
end end
...@@ -56,15 +46,65 @@ defp search_query(query_string, for_user, following) do ...@@ -56,15 +46,65 @@ defp search_query(query_string, for_user, following) do
|> base_query(following) |> base_query(following)
|> filter_blocked_user(for_user) |> filter_blocked_user(for_user)
|> filter_blocked_domains(for_user) |> filter_blocked_domains(for_user)
|> search_subqueries(query_string) |> fts_search(query_string)
|> union_subqueries |> trigram_rank(query_string)
|> distinct_query() |> boost_search_rank(for_user)
|> boost_search_rank_query(for_user)
|> subquery() |> subquery()
|> order_by(desc: :search_rank) |> order_by(desc: :search_rank)
|> maybe_restrict_local(for_user) |> maybe_restrict_local(for_user)
end end
@nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
defp fts_search(query, query_string) do
{nickname_weight, name_weight} =
if String.match?(query_string, @nickname_regex) do
{"A", "B"}
else
{"B", "A"}
end
query_string = to_tsquery(query_string)
from(
u in query,
where:
fragment(
"""
(setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?)
""",
u.name,
^name_weight,
u.nickname,
^nickname_weight,
^query_string
)
)
end
defp to_tsquery(query_string) do
String.trim_trailing(query_string, "@" <> local_domain())
|> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
|> String.trim()
|> String.split()
|> Enum.map(&(&1 <> ":*"))
|> Enum.join(" | ")
end
defp trigram_rank(query, query_string) do
from(
u in query,
select_merge: %{
search_rank:
fragment(
"similarity(?, trim(? || ' ' || coalesce(?, '')))",
^query_string,
u.nickname,
u.name
)
}
)
end
defp base_query(_user, false), do: User defp base_query(_user, false), do: User
defp base_query(user, true), do: User.get_followers_query(user) defp base_query(user, true), do: User.get_followers_query(user)
...@@ -87,21 +127,6 @@ defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}}) ...@@ -87,21 +127,6 @@ defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}})
defp filter_blocked_domains(query, _), do: query defp filter_blocked_domains(query, _), do: query
defp union_subqueries({fts_subquery, trigram_subquery}) do
from(s in trigram_subquery, union_all: ^fts_subquery)
end
defp search_subqueries(base_query, query_string) do
{
fts_search_subquery(base_query, query_string),
trigram_search_subquery(base_query, query_string)
}
end
defp distinct_query(q) do
from(s in subquery(q), order_by: s.search_type, distinct: s.id)
end
defp maybe_resolve(true, user, query) do defp maybe_resolve(true, user, query) do
case {limit(), user} do case {limit(), user} do
{:all, _} -> :noop {:all, _} -> :noop
...@@ -126,9 +151,9 @@ defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauth ...@@ -126,9 +151,9 @@ defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauth
defp restrict_local(q), do: where(q, [u], u.local == true) defp restrict_local(q), do: where(q, [u], u.local == true)
defp boost_search_rank_query(query, nil), do: query defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
defp boost_search_rank_query(query, for_user) do defp boost_search_rank(query, %User{} = for_user) do
friends_ids = User.get_friends_ids(for_user) friends_ids = User.get_friends_ids(for_user)
followers_ids = User.get_followers_ids(for_user) followers_ids = User.get_followers_ids(for_user)
...@@ -137,8 +162,8 @@ defp boost_search_rank_query(query, for_user) do ...@@ -137,8 +162,8 @@ defp boost_search_rank_query(query, for_user) do
search_rank: search_rank:
fragment( fragment(
""" """
CASE WHEN (?) THEN 0.5 + (?) * 1.3 CASE WHEN (?) THEN (?) * 1.5
WHEN (?) THEN 0.5 + (?) * 1.2 WHEN (?) THEN (?) * 1.3
WHEN (?) THEN (?) * 1.1 WHEN (?) THEN (?) * 1.1
ELSE (?) END ELSE (?) END
""", """,
...@@ -154,70 +179,5 @@ defp boost_search_rank_query(query, for_user) do ...@@ -154,70 +179,5 @@ defp boost_search_rank_query(query, for_user) do
) )
end end
@spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() defp boost_search_rank(query, _for_user), do: query
defp fts_search_subquery(query, term) do
processed_query =
String.trim_trailing(term, "@" <> local_domain())
|> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
|> String.trim()
|> String.split()
|> Enum.map(&(&1 <> ":*"))
|> Enum.join(" | ")
from(
u in query,
select_merge: %{
search_type: ^0,
search_rank:
fragment(
"""
ts_rank_cd(
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
to_tsquery('simple', ?),
32
)
""",
u.nickname,
u.name,
^processed_query
)
},
where:
fragment(
"""
(setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
""",
u.nickname,
u.name,
^processed_query
)
)
|> User.restrict_deactivated()
end
@spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
defp trigram_search_subquery(query, term) do
term = String.trim_trailing(term, "@" <> local_domain())
from(
u in query,
select_merge: %{
# ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
search_type: fragment("?", 1),
search_rank:
fragment(
"similarity(?, trim(? || ' ' || coalesce(?, '')))",
^term,
u.nickname,
u.name
)
},
where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
)
|> User.restrict_deactivated()
end
defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
end end
...@@ -349,7 +349,7 @@ def update_follow_state_for_all( ...@@ -349,7 +349,7 @@ def update_follow_state_for_all(
try do try do
Ecto.Adapters.SQL.query!( Ecto.Adapters.SQL.query!(
Repo, Repo,
"UPDATE activities SET data = jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'", "UPDATE activities SET data = safe_jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'",
[state, actor, object] [state, actor, object]
) )
......
...@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do ...@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
def project do def project do
[ [
app: :pleroma, app: :pleroma,
version: version("1.1.0"), version: version("1.1.1"),
elixir: "~> 1.7", elixir: "~> 1.7",
elixirc_paths: elixirc_paths(Mix.env()), elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(), compilers: [:phoenix, :gettext] ++ Mix.compilers(),
......
defmodule Pleroma.Repo.Migrations.CreateSafeJsonbSet do
use Ecto.Migration
alias Pleroma.User
def change do
execute("""
create or replace function safe_jsonb_set(target jsonb, path text[], new_value jsonb, create_missing boolean default true) returns jsonb as $$
declare
result jsonb;
begin
result := jsonb_set(target, path, coalesce(new_value, 'null'::jsonb), create_missing);
if result is NULL then
raise 'jsonb_set tried to wipe the object, please report this incindent to Pleroma bug tracker. https://git.pleroma.social/pleroma/pleroma/issues/new';
return target;
else
return result;
end if;
end;
$$ language plpgsql;
""")
end
end
...@@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do ...@@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do
alias Pleroma.User alias Pleroma.User
def change do def change do
execute("update users set info = jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true") execute("update users set info = safe_jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true")
end end
end end
defmodule Pleroma.SafeJsonbSetTest do
use Pleroma.DataCase
test "it doesn't wipe the object when asked to set the value to NULL" do
assert %{rows: [[%{"key" => "value", "test" => nil}]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select safe_jsonb_set('{\"key\": \"value\"}'::jsonb, '{test}', NULL);",
[]
)
end
end
...@@ -65,21 +65,6 @@ test "finds users, considering density of matched tokens" do ...@@ -65,21 +65,6 @@ test "finds users, considering density of matched tokens" do
assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id)
end end
test "finds users, ranking by similarity" do
u1 = insert(:user, %{name: "lain"})
_u2 = insert(:user, %{name: "ean"})
u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"})
u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"})
assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id)
end
test "finds users, handling misspelled requests" do
u1 = insert(:user, %{name: "lain"})
assert [u1.id] == Enum.map(User.search("laiin"), & &1.id)
end
test "finds users, boosting ranks of friends and followers" do test "finds users, boosting ranks of friends and followers" do
u1 = insert(:user) u1 = insert(:user)
u2 = insert(:user, %{name: "Doe"}) u2 = insert(:user, %{name: "Doe"})
...@@ -163,17 +148,6 @@ test "find all users for unauthenticated users when `limit_to_local_content` is ...@@ -163,17 +148,6 @@ test "find all users for unauthenticated users when `limit_to_local_content` is
Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated)
end end
test "finds a user whose name is nil" do
_user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"})
user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"})
assert user_two ==
User.search("lain@pleroma.soykaf.com")
|> List.first()
|> Map.put(:search_rank, nil)
|> Map.put(:search_type, nil)
end
test "does not yield false-positive matches" do test "does not yield false-positive matches" do
insert(:user, %{name: "John Doe"}) insert(:user, %{name: "John Doe"})
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment