Merge branch 'feature/optional-tld-validation' into 'master'

Add an option to disable TLD validation

Closes #8

See merge request pleroma/auto_linker!17

lib/auto_linker.ex

@@ -48,6 +48,7 @@ defmodule AutoLinker do
   * `hashtag_prefix: nil` - a prefix to build a link for a hashtag (example: `https://example.com/tag/`)
   * `hashtag_handler: nil` - a custom handler to validate and formart a hashtag
   * `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.)
+  * `validate_tld: true` - Set to false to disable TLD validation for urls/emails, also can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)
 
   Each of the above options can be specified when calling `link(text, opts)`
   or can be set in the `:auto_linker`'s configuration. For example:

lib/auto_linker/parser.ex

@@ -42,7 +42,7 @@ defmodule AutoLinker.Parser do
 
   @tlds "./priv/tlds.txt" |> File.read!() |> String.split("\n", trim: true) |> MapSet.new()
 
-  @default_opts ~w(url)a
+  @default_opts ~w(url validate_tld)a
 
   @doc """
   Parse the given string, identifying items to link.
@@ -262,7 +262,7 @@ defmodule AutoLinker.Parser do
   def check_and_link(buffer, opts, _user_acc) do
     str = strip_parens(buffer)
 
-    if url?(str, opts[:scheme]) do
+    if url?(str, opts) do
       case parse_link(str, opts) do
         ^buffer -> link_url(buffer, opts)
         url -> String.replace(buffer, url, link_url(url, opts))
@@ -285,7 +285,7 @@ defmodule AutoLinker.Parser do
   defp strip_parens(buffer), do: buffer
 
   def check_and_link_email(buffer, opts, _user_acc) do
-    if email?(buffer), do: link_email(buffer, opts), else: buffer
+    if email?(buffer, opts), do: link_email(buffer, opts), else: buffer
   end
 
   def check_and_link_phone(buffer, opts, _user_acc) do
@@ -307,7 +307,7 @@ defmodule AutoLinker.Parser do
   end
 
   def check_and_link_extra("xmpp:" <> handle, opts, _user_acc) do
-    if email?(handle), do: link_extra("xmpp:" <> handle, opts), else: handle
+    if email?(handle, opts), do: link_extra("xmpp:" <> handle, opts), else: handle
   end
 
   def check_and_link_extra(buffer, opts, _user_acc) do
@@ -315,30 +315,40 @@ defmodule AutoLinker.Parser do
   end
 
   # @doc false
-  def url?(buffer, true) do
-    valid_url?(buffer) && Regex.match?(@match_scheme, buffer) && valid_tld?(buffer)
-  end
 
-  def url?(buffer, _) do
-    valid_url?(buffer) && Regex.match?(@match_url, buffer) && valid_tld?(buffer)
+  def url?(buffer, opts) do
+    if opts[:scheme] do
+      valid_url?(buffer) && Regex.match?(@match_scheme, buffer) && valid_tld?(buffer, opts)
+    else
+      valid_url?(buffer) && Regex.match?(@match_url, buffer) && valid_tld?(buffer, opts)
+    end
   end
 
-  def email?(buffer) do
-    valid_url?(buffer) && Regex.match?(@match_email, buffer) && valid_tld?(buffer)
+  def email?(buffer, opts) do
+    valid_url?(buffer) && Regex.match?(@match_email, buffer) && valid_tld?(buffer, opts)
   end
 
   defp valid_url?(url), do: !Regex.match?(@invalid_url, url)
 
-  def valid_tld?(buffer) do
-    with [host] <- Regex.run(@match_hostname, buffer, capture: [:host]) do
-      if ip?(host) do
+  def valid_tld?(buffer, opts) do
+    cond do
+      opts[:validate_tld] == false ->
         true
-      else
-        tld = host |> String.split(".") |> List.last()
-        MapSet.member?(@tlds, tld)
-      end
-    else
-      _ -> false
+
+      opts[:validate_tld] == :no_scheme && opts[:scheme] ->
+        true
+
+      true ->
+        with [host] <- Regex.run(@match_hostname, buffer, capture: [:host]) do
+          if ip?(host) do
+            true
+          else
+            tld = host |> String.split(".") |> List.last()
+            MapSet.member?(@tlds, tld)
+          end
+        else
+          _ -> false
+        end
     end
   end
 

test/parser_test.exs

@@ -8,28 +8,100 @@ defmodule AutoLinker.ParserTest do
     test "valid scheme true" do
       valid_scheme_urls()
       |> Enum.each(fn url ->
-        assert url?(url, true)
+        assert url?(url, scheme: true, validate_tld: true)
       end)
     end
 
     test "invalid scheme true" do
       invalid_scheme_urls()
       |> Enum.each(fn url ->
-        refute url?(url, true)
+        refute url?(url, scheme: true, validate_tld: true)
       end)
     end
 
     test "valid scheme false" do
       valid_non_scheme_urls()
       |> Enum.each(fn url ->
-        assert url?(url, false)
+        assert url?(url, scheme: false, validate_tld: true)
       end)
     end
 
     test "invalid scheme false" do
       invalid_non_scheme_urls()
       |> Enum.each(fn url ->
-        refute url?(url, false)
+        refute url?(url, scheme: false, validate_tld: true)
+      end)
+    end
+
+    test "checks the tld for url with a scheme when validate_tld: true" do
+      custom_tld_scheme_urls()
+      |> Enum.each(fn url ->
+        refute url?(url, scheme: true, validate_tld: true)
+      end)
+    end
+
+    test "does not check the tld for url with a scheme when validate_tld: false" do
+      custom_tld_scheme_urls()
+      |> Enum.each(fn url ->
+        assert url?(url, scheme: true, validate_tld: false)
+      end)
+    end
+
+    test "does not check the tld for url with a scheme when validate_tld: :no_scheme" do
+      custom_tld_scheme_urls()
+      |> Enum.each(fn url ->
+        assert url?(url, scheme: true, validate_tld: :no_scheme)
+      end)
+    end
+
+    test "checks the tld for url without a scheme when validate_tld: true" do
+      custom_tld_non_scheme_urls()
+      |> Enum.each(fn url ->
+        refute url?(url, scheme: false, validate_tld: true)
+      end)
+    end
+
+    test "checks the tld for url without a scheme when validate_tld: :no_scheme" do
+      custom_tld_non_scheme_urls()
+      |> Enum.each(fn url ->
+        refute url?(url, scheme: false, validate_tld: :no_scheme)
+      end)
+    end
+
+    test "does not check the tld for url without a scheme when validate_tld: false" do
+      custom_tld_non_scheme_urls()
+      |> Enum.each(fn url ->
+        assert url?(url, scheme: false, validate_tld: false)
+      end)
+    end
+  end
+
+  describe "email?" do
+    test "identifies valid emails" do
+      valid_emails()
+      |> Enum.each(fn email ->
+        assert email?(email, [])
+      end)
+    end
+
+    test "identifies invalid emails" do
+      invalid_emails()
+      |> Enum.each(fn email ->
+        refute email?(email, [])
+      end)
+    end
+
+    test "does not validate tlds when validate_tld: false" do
+      valid_custom_tld_emails()
+      |> Enum.each(fn email ->
+        assert email?(email, validate_tld: false)
+      end)
+    end
+
+    test "validates tlds when validate_tld: true" do
+      valid_custom_tld_emails()
+      |> Enum.each(fn email ->
+        refute email?(email, validate_tld: true)
       end)
     end
   end
@@ -216,4 +288,24 @@ defmodule AutoLinker.ParserTest do
       "x5",
       "(555) 555-55"
     ]
+
+  def custom_tld_scheme_urls,
+    do: [
+      "http://whatever.null/",
+      "https://example.o/index.html",
+      "http://pleroma.i2p/test",
+      "http://misskey.loki"
+    ]
+
+  def custom_tld_non_scheme_urls,
+    do: [
+      "whatever.null/",
+      "example.o/index.html",
+      "pleroma.i2p/test",
+      "misskey.loki"
+    ]
+
+  def valid_emails, do: ["rms@ai.mit.edu", "vc@cock.li"]
+  def invalid_emails, do: ["rms[at]ai.mit.edu", "vc@cock", "xmpp:lain@trashserver.net"]
+  def valid_custom_tld_emails, do: ["guardian@33y6fjyhs3phzfjj.onion", "hi@company.null"]
 end