Merge branch 'bugfix/br-vs-newline' into 'develop'

Formatting: Do not use \n and prefer <br> instead

Closes #1374 and #1375

See merge request pleroma/pleroma!2204

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [unreleased]
+### Changed
+- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `<br/>` for newlines
+
 ## [2.0.0] - 2019-03-08
 ### Security
 - Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.

lib/pleroma/earmark_renderer.ex

@@ -0,0 +1,256 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+#
+# This file is derived from Earmark, under the following copyright:
+# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
+# SPDX-License-Identifier: Apache-2.0
+# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
+defmodule Pleroma.EarmarkRenderer do
+  @moduledoc false
+
+  alias Earmark.Block
+  alias Earmark.Context
+  alias Earmark.HtmlRenderer
+  alias Earmark.Options
+
+  import Earmark.Inline, only: [convert: 3]
+  import Earmark.Helpers.HtmlHelpers
+  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
+  import Earmark.Context, only: [append: 2, set_value: 2]
+  import Earmark.Options, only: [get_mapper: 1]
+
+  @doc false
+  def render(blocks, %Context{options: %Options{}} = context) do
+    messages = get_messages(context)
+
+    {contexts, html} =
+      get_mapper(context.options).(
+        blocks,
+        &render_block(&1, put_in(context.options.messages, []))
+      )
+      |> Enum.unzip()
+
+    all_messages =
+      contexts
+      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
+
+    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
+  end
+
+  #############
+  # Paragraph #
+  #############
+  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
+    lines = convert(lines, lnb, context)
+    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
+  end
+
+  ########
+  # Html #
+  ########
+  defp render_block(%Block.Html{html: html}, context) do
+    {context, html}
+  end
+
+  defp render_block(%Block.HtmlComment{lines: lines}, context) do
+    {context, lines}
+  end
+
+  defp render_block(%Block.HtmlOneline{html: html}, context) do
+    {context, html}
+  end
+
+  #########
+  # Ruler #
+  #########
+  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
+    add_attrs(context, "<hr />", attrs, [], lnb)
+  end
+
+  ###########
+  # Heading #
+  ###########
+  defp render_block(
+         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
+         context
+       ) do
+    converted = convert(content, lnb, context)
+    html = "<h#{level}>#{converted.value}</h#{level}>"
+    add_attrs(converted, html, attrs, [], lnb)
+  end
+
+  ##############
+  # Blockquote #
+  ##############
+
+  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, body} = render(blocks, context)
+    html = "<blockquote>#{body}</blockquote>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  #########
+  # Table #
+  #########
+
+  defp render_block(
+         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
+         context
+       ) do
+    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
+    context2 = set_value(context1, html)
+
+    context3 =
+      if header do
+        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
+      else
+        # Maybe an error, needed append(context, html)
+        context2
+      end
+
+    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
+
+    {context4, [context4.value, "</table>"]}
+  end
+
+  ########
+  # Code #
+  ########
+
+  defp render_block(
+         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
+         %Context{options: options} = context
+       ) do
+    class =
+      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
+
+    tag = ~s[<pre><code#{class}>]
+    lines = options.render_code.(block)
+    html = ~s[#{tag}#{lines}</code></pre>]
+    add_attrs(context, html, attrs, [], lnb)
+  end
+
+  #########
+  # Lists #
+  #########
+
+  defp render_block(
+         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
+         context
+       ) do
+    {context1, content} = render(items, context)
+    html = "<#{type}#{start}>#{content}</#{type}>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a single paragraph list item, and remove the para tags
+  defp render_block(
+         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
+         context
+       )
+       when length(blocks) == 1 do
+    {context1, content} = render(blocks, context)
+    content = Regex.replace(~r{</?p>}, content, "")
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a spaced list item
+  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, content} = render(blocks, context)
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  ##################
+  # Footnote Block #
+  ##################
+
+  defp render_block(%Block.FnList{blocks: footnotes}, context) do
+    items =
+      Enum.map(footnotes, fn note ->
+        blocks = append_footnote_link(note)
+        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
+      end)
+
+    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
+    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
+  end
+
+  #######################################
+  # Isolated IALs are rendered as paras #
+  #######################################
+
+  defp render_block(%Block.Ial{verbatim: verbatim}, context) do
+    {context, "<p>{:#{verbatim}}</p>"}
+  end
+
+  ####################
+  # IDDef is ignored #
+  ####################
+
+  defp render_block(%Block.IdDef{}, context), do: {context, ""}
+
+  #####################################
+  # And here are the inline renderers #
+  #####################################
+
+  defdelegate br, to: HtmlRenderer
+  defdelegate codespan(text), to: HtmlRenderer
+  defdelegate em(text), to: HtmlRenderer
+  defdelegate strong(text), to: HtmlRenderer
+  defdelegate strikethrough(text), to: HtmlRenderer
+
+  defdelegate link(url, text), to: HtmlRenderer
+  defdelegate link(url, text, title), to: HtmlRenderer
+
+  defdelegate image(path, alt, title), to: HtmlRenderer
+
+  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
+
+  # Table rows
+  defp add_trs(context, rows, tag, aligns, lnb) do
+    numbered_rows =
+      rows
+      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
+
+    numbered_rows
+    |> Enum.reduce(context, fn {row, lnb}, ctx ->
+      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
+    end)
+  end
+
+  defp add_tds(context, row, tag, aligns, lnb) do
+    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
+  end
+
+  defp add_td_fn(row, tag, aligns, lnb) do
+    fn n, ctx ->
+      style =
+        case Enum.at(aligns, n - 1, :default) do
+          :default -> ""
+          align -> " style=\"text-align: #{align}\""
+        end
+
+      col = Enum.at(row, n - 1)
+      converted = convert(col, lnb, set_messages(ctx, []))
+      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
+    end
+  end
+
+  ###############################
+  # Append Footnote Return Link #
+  ###############################
+
+  defdelegate append_footnote_link(note), to: HtmlRenderer
+  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
+
+  defdelegate render_code(lines), to: HtmlRenderer
+
+  defp code_classes(language, prefix) do
+    ["" | String.split(prefix || "")]
+    |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
+    |> Enum.join(" ")
+  end
+end

lib/pleroma/web/common_api/utils.ex

@@ -331,7 +331,7 @@ def format_input(text, "text/html", options) do
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
-    |> Earmark.as_html!()
+    |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
     |> Formatter.linkify(options)
     |> Formatter.html_escape("text/html")
   end

mix.exs

@@ -126,7 +126,7 @@ defp deps do
       {:ex_aws_s3, "~> 2.0"},
       {:sweet_xml, "~> 0.6.6"},
       {:earmark, "~> 1.3"},
-      {:bbcode, "~> 0.1.1"},
+      {:bbcode_pleroma, "~> 0.2.0"},
       {:ex_machina, "~> 2.3", only: :test},
       {:credo, "~> 1.1.0", only: [:dev, :test], runtime: false},
       {:mock, "~> 0.3.3", only: :test},

mix.lock

@@ -3,10 +3,11 @@
   "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]},
   "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
-  "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5a981b98ac7d366a9b6bf40eac389aaf4d6e623c631e6b6f8a6b571efaafd338"},
+  "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]},
+  "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
   "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
-  "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"},
+  "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
   "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"},
   "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]},
   "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
@@ -110,4 +111,3 @@
   "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "9315c8f37c108835cf3f8e9157d7a9b8f420a34f402d1b1620a31aed5b93ecdf"},
   "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []},
 }
-

test/earmark_renderer_test.ex

@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.EarmarkRendererTest do
+  use ExUnit.Case
+
+  test "Paragraph" do
+    code = ~s[Hello\n\nWorld!]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<p>Hello</p><p>World!</p>"
+  end
+
+  test "raw HTML" do
+    code = ~s[<a href="http://example.org/">OwO</a><!-- what's this?-->]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<p>#{code}</p>"
+  end
+
+  test "rulers" do
+    code = ~s[before\n\n-----\n\nafter]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<p>before</p><hr /><p>after</p>"
+  end
+
+  test "headings" do
+    code = ~s[# h1\n## h2\n### h3\n]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<h1>h1</h1><h2>h2</h2><h3>h3</h3>]
+  end
+
+  test "blockquote" do
+    code = ~s[> whoms't are you quoting?]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<blockquote><p>whoms’t are you quoting?</p></blockquote>"
+  end
+
+  test "code" do
+    code = ~s[`mix`]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<p><code class="inline">mix</code></p>]
+
+    code = ~s[``mix``]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<p><code class="inline">mix</code></p>]
+
+    code = ~s[```\nputs "Hello World"\n```]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<pre><code class="">puts &quot;Hello World&quot;</code></pre>]
+  end
+
+  test "lists" do
+    code = ~s[- one\n- two\n- three\n- four]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>"
+
+    code = ~s[1. one\n2. two\n3. three\n4. four\n]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<ol><li>one</li><li>two</li><li>three</li><li>four</li></ol>"
+  end
+
+  test "delegated renderers" do
+    code = ~s[a<br/>b]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == "<p>#{code}</p>"
+
+    code = ~s[*aaaa~*]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<p><em>aaaa~</em></p>]
+
+    code = ~s[**aaaa~**]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<p><strong>aaaa~</strong></p>]
+
+    # strikethrought
+    code = ~s[<del>aaaa~</del>]
+    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    assert result == ~s[<p><del>aaaa~</del></p>]
+  end
+end

test/web/common_api/common_api_utils_test.exs

@@ -89,8 +89,8 @@ test "works for bare text/html" do
 
       assert output == expected
 
-      text = "<p>hello world!</p>\n\n<p>second paragraph</p>"
+      text = "<p>hello world!</p><br/>\n<p>second paragraph</p>"
-      expected = "<p>hello world!</p>\n\n<p>second paragraph</p>"
+      expected = "<p>hello world!</p><br/>\n<p>second paragraph</p>"
 
       {output, [], []} = Utils.format_input(text, "text/html")
 
@@ -99,14 +99,14 @@ test "works for bare text/html" do
 
     test "works for bare text/markdown" do
       text = "**hello world**"
-      expected = "<p><strong>hello world</strong></p>\n"
+      expected = "<p><strong>hello world</strong></p>"
 
       {output, [], []} = Utils.format_input(text, "text/markdown")
 
       assert output == expected
 
       text = "**hello world**\n\n*another paragraph*"
-      expected = "<p><strong>hello world</strong></p>\n<p><em>another paragraph</em></p>\n"
+      expected = "<p><strong>hello world</strong></p><p><em>another paragraph</em></p>"
 
       {output, [], []} = Utils.format_input(text, "text/markdown")
 
@@ -118,7 +118,7 @@ test "works for bare text/markdown" do
       by someone
       """
 
-      expected = "<blockquote><p>cool quote</p>\n</blockquote>\n<p>by someone</p>\n"
+      expected = "<blockquote><p>cool quote</p></blockquote><p>by someone</p>"
 
       {output, [], []} = Utils.format_input(text, "text/markdown")
 
@@ -134,7 +134,7 @@ test "works for bare text/bbcode" do
       assert output == expected
 
       text = "[b]hello world![/b]\n\nsecond paragraph!"
-      expected = "<strong>hello world!</strong><br>\n<br>\nsecond paragraph!"
+      expected = "<strong>hello world!</strong><br><br>second paragraph!"
 
       {output, [], []} = Utils.format_input(text, "text/bbcode")
 
@@ -143,7 +143,7 @@ test "works for bare text/bbcode" do
       text = "[b]hello world![/b]\n\n<strong>second paragraph!</strong>"
 
       expected =
-        "<strong>hello world!</strong><br>\n<br>\n&lt;strong&gt;second paragraph!&lt;/strong&gt;"
+        "<strong>hello world!</strong><br><br>&lt;strong&gt;second paragraph!&lt;/strong&gt;"
 
       {output, [], []} = Utils.format_input(text, "text/bbcode")
 
@@ -156,16 +156,14 @@ test "works for text/markdown with mentions" do
 
       text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*"
 
-      expected =
-        ~s(<p><strong>hello world</strong></p>\n<p><em>another <span class="h-card"><a data-user="#{
-          user.id
-        }" class="u-url mention" href="http://foo.com/user__test" rel="ugc">@<span>user__test</span></a></span> and <span class="h-card"><a data-user="#{
-          user.id
-        }" class="u-url mention" href="http://foo.com/user__test" rel="ugc">@<span>user__test</span></a></span> <a href="http://google.com" rel="ugc">google.com</a> paragraph</em></p>\n)
-
       {output, _, _} = Utils.format_input(text, "text/markdown")
 
-      assert output == expected
+      assert output ==
+               ~s(<p><strong>hello world</strong></p><p><em>another <span class="h-card"><a data-user="#{
+                 user.id
+               }" class="u-url mention" href="http://foo.com/user__test" rel="ugc">@<span>user__test</span></a></span> and <span class="h-card"><a data-user="#{
+                 user.id
+               }" class="u-url mention" href="http://foo.com/user__test" rel="ugc">@<span>user__test</span></a></span> <a href="http://google.com" rel="ugc">google.com</a> paragraph</em></p>)
     end
   end