Commit 95e81884 authored by minibikini's avatar minibikini

Merge branch 'feature/optional-tld-validation' into 'master'

Add an option to disable TLD validation

Closes #8

See merge request pleroma/auto_linker!17
parents e2385402 70cbfdc8
Pipeline #13432 passed with stages
in 1 minute and 59 seconds
...@@ -48,6 +48,7 @@ defmodule AutoLinker do ...@@ -48,6 +48,7 @@ defmodule AutoLinker do
* `hashtag_prefix: nil` - a prefix to build a link for a hashtag (example: `https://example.com/tag/`) * `hashtag_prefix: nil` - a prefix to build a link for a hashtag (example: `https://example.com/tag/`)
* `hashtag_handler: nil` - a custom handler to validate and formart a hashtag * `hashtag_handler: nil` - a custom handler to validate and formart a hashtag
* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) * `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.)
* `validate_tld: true` - Set to false to disable TLD validation for urls/emails, also can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)
Each of the above options can be specified when calling `link(text, opts)` Each of the above options can be specified when calling `link(text, opts)`
or can be set in the `:auto_linker`'s configuration. For example: or can be set in the `:auto_linker`'s configuration. For example:
......
...@@ -42,7 +42,7 @@ defmodule AutoLinker.Parser do ...@@ -42,7 +42,7 @@ defmodule AutoLinker.Parser do
@tlds "./priv/tlds.txt" |> File.read!() |> String.split("\n", trim: true) |> MapSet.new() @tlds "./priv/tlds.txt" |> File.read!() |> String.split("\n", trim: true) |> MapSet.new()
@default_opts ~w(url)a @default_opts ~w(url validate_tld)a
@doc """ @doc """
Parse the given string, identifying items to link. Parse the given string, identifying items to link.
...@@ -262,7 +262,7 @@ defmodule AutoLinker.Parser do ...@@ -262,7 +262,7 @@ defmodule AutoLinker.Parser do
def check_and_link(buffer, opts, _user_acc) do def check_and_link(buffer, opts, _user_acc) do
str = strip_parens(buffer) str = strip_parens(buffer)
if url?(str, opts[:scheme]) do if url?(str, opts) do
case parse_link(str, opts) do case parse_link(str, opts) do
^buffer -> link_url(buffer, opts) ^buffer -> link_url(buffer, opts)
url -> String.replace(buffer, url, link_url(url, opts)) url -> String.replace(buffer, url, link_url(url, opts))
...@@ -285,7 +285,7 @@ defmodule AutoLinker.Parser do ...@@ -285,7 +285,7 @@ defmodule AutoLinker.Parser do
defp strip_parens(buffer), do: buffer defp strip_parens(buffer), do: buffer
def check_and_link_email(buffer, opts, _user_acc) do def check_and_link_email(buffer, opts, _user_acc) do
if email?(buffer), do: link_email(buffer, opts), else: buffer if email?(buffer, opts), do: link_email(buffer, opts), else: buffer
end end
def check_and_link_phone(buffer, opts, _user_acc) do def check_and_link_phone(buffer, opts, _user_acc) do
...@@ -307,7 +307,7 @@ defmodule AutoLinker.Parser do ...@@ -307,7 +307,7 @@ defmodule AutoLinker.Parser do
end end
def check_and_link_extra("xmpp:" <> handle, opts, _user_acc) do def check_and_link_extra("xmpp:" <> handle, opts, _user_acc) do
if email?(handle), do: link_extra("xmpp:" <> handle, opts), else: handle if email?(handle, opts), do: link_extra("xmpp:" <> handle, opts), else: handle
end end
def check_and_link_extra(buffer, opts, _user_acc) do def check_and_link_extra(buffer, opts, _user_acc) do
...@@ -315,30 +315,40 @@ defmodule AutoLinker.Parser do ...@@ -315,30 +315,40 @@ defmodule AutoLinker.Parser do
end end
# @doc false # @doc false
def url?(buffer, true) do
valid_url?(buffer) && Regex.match?(@match_scheme, buffer) && valid_tld?(buffer)
end
def url?(buffer, _) do def url?(buffer, opts) do
valid_url?(buffer) && Regex.match?(@match_url, buffer) && valid_tld?(buffer) if opts[:scheme] do
valid_url?(buffer) && Regex.match?(@match_scheme, buffer) && valid_tld?(buffer, opts)
else
valid_url?(buffer) && Regex.match?(@match_url, buffer) && valid_tld?(buffer, opts)
end
end end
def email?(buffer) do def email?(buffer, opts) do
valid_url?(buffer) && Regex.match?(@match_email, buffer) && valid_tld?(buffer) valid_url?(buffer) && Regex.match?(@match_email, buffer) && valid_tld?(buffer, opts)
end end
defp valid_url?(url), do: !Regex.match?(@invalid_url, url) defp valid_url?(url), do: !Regex.match?(@invalid_url, url)
def valid_tld?(buffer) do def valid_tld?(buffer, opts) do
with [host] <- Regex.run(@match_hostname, buffer, capture: [:host]) do cond do
if ip?(host) do opts[:validate_tld] == false ->
true true
else
tld = host |> String.split(".") |> List.last() opts[:validate_tld] == :no_scheme && opts[:scheme] ->
MapSet.member?(@tlds, tld) true
end
else true ->
_ -> false with [host] <- Regex.run(@match_hostname, buffer, capture: [:host]) do
if ip?(host) do
true
else
tld = host |> String.split(".") |> List.last()
MapSet.member?(@tlds, tld)
end
else
_ -> false
end
end end
end end
......
...@@ -8,28 +8,100 @@ defmodule AutoLinker.ParserTest do ...@@ -8,28 +8,100 @@ defmodule AutoLinker.ParserTest do
test "valid scheme true" do test "valid scheme true" do
valid_scheme_urls() valid_scheme_urls()
|> Enum.each(fn url -> |> Enum.each(fn url ->
assert url?(url, true) assert url?(url, scheme: true, validate_tld: true)
end) end)
end end
test "invalid scheme true" do test "invalid scheme true" do
invalid_scheme_urls() invalid_scheme_urls()
|> Enum.each(fn url -> |> Enum.each(fn url ->
refute url?(url, true) refute url?(url, scheme: true, validate_tld: true)
end) end)
end end
test "valid scheme false" do test "valid scheme false" do
valid_non_scheme_urls() valid_non_scheme_urls()
|> Enum.each(fn url -> |> Enum.each(fn url ->
assert url?(url, false) assert url?(url, scheme: false, validate_tld: true)
end) end)
end end
test "invalid scheme false" do test "invalid scheme false" do
invalid_non_scheme_urls() invalid_non_scheme_urls()
|> Enum.each(fn url -> |> Enum.each(fn url ->
refute url?(url, false) refute url?(url, scheme: false, validate_tld: true)
end)
end
test "checks the tld for url with a scheme when validate_tld: true" do
custom_tld_scheme_urls()
|> Enum.each(fn url ->
refute url?(url, scheme: true, validate_tld: true)
end)
end
test "does not check the tld for url with a scheme when validate_tld: false" do
custom_tld_scheme_urls()
|> Enum.each(fn url ->
assert url?(url, scheme: true, validate_tld: false)
end)
end
test "does not check the tld for url with a scheme when validate_tld: :no_scheme" do
custom_tld_scheme_urls()
|> Enum.each(fn url ->
assert url?(url, scheme: true, validate_tld: :no_scheme)
end)
end
test "checks the tld for url without a scheme when validate_tld: true" do
custom_tld_non_scheme_urls()
|> Enum.each(fn url ->
refute url?(url, scheme: false, validate_tld: true)
end)
end
test "checks the tld for url without a scheme when validate_tld: :no_scheme" do
custom_tld_non_scheme_urls()
|> Enum.each(fn url ->
refute url?(url, scheme: false, validate_tld: :no_scheme)
end)
end
test "does not check the tld for url without a scheme when validate_tld: false" do
custom_tld_non_scheme_urls()
|> Enum.each(fn url ->
assert url?(url, scheme: false, validate_tld: false)
end)
end
end
describe "email?" do
test "identifies valid emails" do
valid_emails()
|> Enum.each(fn email ->
assert email?(email, [])
end)
end
test "identifies invalid emails" do
invalid_emails()
|> Enum.each(fn email ->
refute email?(email, [])
end)
end
test "does not validate tlds when validate_tld: false" do
valid_custom_tld_emails()
|> Enum.each(fn email ->
assert email?(email, validate_tld: false)
end)
end
test "validates tlds when validate_tld: true" do
valid_custom_tld_emails()
|> Enum.each(fn email ->
refute email?(email, validate_tld: true)
end) end)
end end
end end
...@@ -216,4 +288,24 @@ defmodule AutoLinker.ParserTest do ...@@ -216,4 +288,24 @@ defmodule AutoLinker.ParserTest do
"x5", "x5",
"(555) 555-55" "(555) 555-55"
] ]
def custom_tld_scheme_urls,
do: [
"http://whatever.null/",
"https://example.o/index.html",
"http://pleroma.i2p/test",
"http://misskey.loki"
]
def custom_tld_non_scheme_urls,
do: [
"whatever.null/",
"example.o/index.html",
"pleroma.i2p/test",
"misskey.loki"
]
def valid_emails, do: ["rms@ai.mit.edu", "vc@cock.li"]
def invalid_emails, do: ["rms[at]ai.mit.edu", "vc@cock", "xmpp:lain@trashserver.net"]
def valid_custom_tld_emails, do: ["guardian@33y6fjyhs3phzfjj.onion", "hi@company.null"]
end end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment