erin/akkoma
1
0
Fork 0
forked from mirrors/akkoma
Code Activity

Merge branch 'earmark' into 'develop'

Browse source 
Upgrade Earmark to v1.4.15

Closes #1942 and #2026

See merge request pleroma/pleroma!3076
This commit is contained in:
feld 2021-05-04 13:29:14 +00:00
parent 377f84f367 dca87c5e7b
commit d5263bfcf3
11 changed files with 131 additions and 342 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Applying ConcurrentLimiter settings via AdminAPI
- User login failures if their `notification_settings` were in a NULL state.
- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
- Fixed some Markdown issues, including trailing slash in links.
## [2.3.0] - 2020-03-01

256
lib/pleroma/earmark_renderer.ex
View file

@ -1,256 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
#
# This file is derived from Earmark, under the following copyright:
# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
# SPDX-License-Identifier: Apache-2.0
# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
defmodule Pleroma.EarmarkRenderer do
@moduledoc false
alias Earmark.Block
alias Earmark.Context
alias Earmark.HtmlRenderer
alias Earmark.Options
import Earmark.Inline, only: [convert: 3]
import Earmark.Helpers.HtmlHelpers
import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
import Earmark.Context, only: [append: 2, set_value: 2]
import Earmark.Options, only: [get_mapper: 1]
@doc false
def render(blocks, %Context{options: %Options{}} = context) do
messages = get_messages(context)
{contexts, html} =
get_mapper(context.options).(
blocks,
&render_block(&1, put_in(context.options.messages, []))
)
|> Enum.unzip()
all_messages =
contexts
|> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
{put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
end
#############
# Paragraph #
#############
defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
lines = convert(lines, lnb, context)
add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
end
########
# Html #
########
defp render_block(%Block.Html{html: html}, context) do
{context, html}
end
defp render_block(%Block.HtmlComment{lines: lines}, context) do
{context, lines}
end
defp render_block(%Block.HtmlOneline{html: html}, context) do
{context, html}
end
#########
# Ruler #
#########
defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
add_attrs(context, "<hr />", attrs, [], lnb)
end
###########
# Heading #
###########
defp render_block(
%Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
context
) do
converted = convert(content, lnb, context)
html = "<h#{level}>#{converted.value}</h#{level}>"
add_attrs(converted, html, attrs, [], lnb)
end
##############
# Blockquote #
##############
defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
{context1, body} = render(blocks, context)
html = "<blockquote>#{body}</blockquote>"
add_attrs(context1, html, attrs, [], lnb)
end
#########
# Table #
#########
defp render_block(
%Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
context
) do
{context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
context2 = set_value(context1, html)
context3 =
if header do
append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
else
# Maybe an error, needed append(context, html)
context2
end
context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
{context4, [context4.value, "</table>"]}
end
########
# Code #
########
defp render_block(
%Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
%Context{options: options} = context
) do
class =
if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
tag = ~s[<pre><code#{class}>]
lines = options.render_code.(block)
html = ~s[#{tag}#{lines}</code></pre>]
add_attrs(context, html, attrs, [], lnb)
end
#########
# Lists #
#########
defp render_block(
%Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
context
) do
{context1, content} = render(items, context)
html = "<#{type}#{start}>#{content}</#{type}>"
add_attrs(context1, html, attrs, [], lnb)
end
# format a single paragraph list item, and remove the para tags
defp render_block(
%Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
context
)
when length(blocks) == 1 do
{context1, content} = render(blocks, context)
content = Regex.replace(~r{</?p>}, content, "")
html = "<li>#{content}</li>"
add_attrs(context1, html, attrs, [], lnb)
end
# format a spaced list item
defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
{context1, content} = render(blocks, context)
html = "<li>#{content}</li>"
add_attrs(context1, html, attrs, [], lnb)
end
##################
# Footnote Block #
##################
defp render_block(%Block.FnList{blocks: footnotes}, context) do
items =
Enum.map(footnotes, fn note ->
blocks = append_footnote_link(note)
%Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
end)
{context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
{context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
end
#######################################
# Isolated IALs are rendered as paras #
#######################################
defp render_block(%Block.Ial{verbatim: verbatim}, context) do
{context, "<p>{:#{verbatim}}</p>"}
end
####################
# IDDef is ignored #
####################
defp render_block(%Block.IdDef{}, context), do: {context, ""}
#####################################
# And here are the inline renderers #
#####################################
defdelegate br, to: HtmlRenderer
defdelegate codespan(text), to: HtmlRenderer
defdelegate em(text), to: HtmlRenderer
defdelegate strong(text), to: HtmlRenderer
defdelegate strikethrough(text), to: HtmlRenderer
defdelegate link(url, text), to: HtmlRenderer
defdelegate link(url, text, title), to: HtmlRenderer
defdelegate image(path, alt, title), to: HtmlRenderer
defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
# Table rows
defp add_trs(context, rows, tag, aligns, lnb) do
numbered_rows =
rows
|> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
numbered_rows
|> Enum.reduce(context, fn {row, lnb}, ctx ->
append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
end)
end
defp add_tds(context, row, tag, aligns, lnb) do
Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
end
defp add_td_fn(row, tag, aligns, lnb) do
fn n, ctx ->
style =
case Enum.at(aligns, n - 1, :default) do
:default -> ""
align -> " style=\"text-align: #{align}\""
end
col = Enum.at(row, n - 1)
converted = convert(col, lnb, set_messages(ctx, []))
append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
end
end
###############################
# Append Footnote Return Link #
###############################
defdelegate append_footnote_link(note), to: HtmlRenderer
defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
defdelegate render_code(lines), to: HtmlRenderer
defp code_classes(language, prefix) do
["" | String.split(prefix || "")]
|> Enum.map(fn pfx -> "#{pfx}#{language}" end)
|> Enum.join(" ")
end
end

4
lib/pleroma/formatter.ex
View file

@ -121,6 +121,10 @@ def mentions_escape(text, options \\ []) do
end
end
def markdown_to_html(text) do
Earmark.as_html!(text, %Earmark.Options{compact_output: true})
end
def html_escape({text, mentions, hashtags}, type) do
{html_escape(text, type), mentions, hashtags}
end

3
lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
View file

@ -5,7 +5,6 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
use Ecto.Schema
alias Pleroma.EarmarkRenderer
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
@ -110,7 +109,7 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
when is_binary(content) do
content =
content
|> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
|> Pleroma.Formatter.markdown_to_html()
|> Pleroma.HTML.filter_tags()
Map.put(data, "content", content)

2
lib/pleroma/web/common_api/utils.ex
View file

@ -286,7 +286,7 @@ def format_input(text, "text/html", options) do
def format_input(text, "text/markdown", options) do
text
|> Formatter.mentions_escape(options)
|> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
|> Formatter.markdown_to_html()
|> Formatter.linkify(options)
|> Formatter.html_escape("text/html")
end

2
mix.exs
View file

@ -144,7 +144,7 @@ defp deps do
{:ex_aws, "~> 2.1.6"},
{:ex_aws_s3, "~> 2.0"},
{:sweet_xml, "~> 0.6.6"},
{:earmark, "1.4.3"},
{:earmark, "1.4.15"},
{:bbcode_pleroma, "~> 0.2.0"},
{:crypt,
git: "https://git.pleroma.social/pleroma/elixir-libraries/crypt.git",

4
mix.lock
View file

@ -27,8 +27,8 @@
"db_connection": {:hex, :db_connection, "2.3.1", "4c9f3ed1ef37471cbdd2762d6655be11e38193904d9c5c1c9389f1b891a3088e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "abaab61780dde30301d840417890bd9f74131041afd02174cf4e10635b3a63f5"},
"decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
"earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"},
"earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"},
"earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"},
"earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
"ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"},
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
"ecto_explain": {:hex, :ecto_explain, "0.1.2", "a9d504cbd4adc809911f796d5ef7ebb17a576a6d32286c3d464c015bd39d5541", [:mix], [], "hexpm", "1d0e7798ae30ecf4ce34e912e5354a0c1c832b7ebceba39298270b9a9f316330"},

3
priv/scrubbers/default.ex
View file

@ -39,6 +39,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.allow_tag_with_these_attributes(:code, [])
Meta.allow_tag_with_these_attributes(:del, [])
Meta.allow_tag_with_these_attributes(:em, [])
Meta.allow_tag_with_these_attributes(:hr, [])
Meta.allow_tag_with_these_attributes(:i, [])
Meta.allow_tag_with_these_attributes(:li, [])
Meta.allow_tag_with_these_attributes(:ol, [])
@ -58,6 +59,8 @@ defmodule Pleroma.HTML.Scrubber.Default do
Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"])
Meta.allow_tag_with_these_attributes(:span, [])
Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"])
@allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images])
if @allow_inline_images do

79
test/pleroma/earmark_renderer_test.exs
View file

@ -1,79 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EarmarkRendererTest do
use Pleroma.DataCase, async: true
test "Paragraph" do
code = ~s[Hello\n\nWorld!]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<p>Hello</p><p>World!</p>"
end
test "raw HTML" do
code = ~s[<a href="http://example.org/">OwO</a><!-- what's this?-->]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<p>#{code}</p>"
end
test "rulers" do
code = ~s[before\n\n-----\n\nafter]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<p>before</p><hr /><p>after</p>"
end
test "headings" do
code = ~s[# h1\n## h2\n### h3\n]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<h1>h1</h1><h2>h2</h2><h3>h3</h3>]
end
test "blockquote" do
code = ~s[> whoms't are you quoting?]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<blockquote><p>whoms’t are you quoting?</p></blockquote>"
end
test "code" do
code = ~s[`mix`]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<p><code class="inline">mix</code></p>]
code = ~s[``mix``]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<p><code class="inline">mix</code></p>]
code = ~s[```\nputs "Hello World"\n```]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<pre><code class="">puts &quot;Hello World&quot;</code></pre>]
end
test "lists" do
code = ~s[- one\n- two\n- three\n- four]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>"
code = ~s[1. one\n2. two\n3. three\n4. four\n]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<ol><li>one</li><li>two</li><li>three</li><li>four</li></ol>"
end
test "delegated renderers" do
code = ~s[a<br/>b]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == "<p>#{code}</p>"
code = ~s[*aaaa~*]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<p><em>aaaa~</em></p>]
code = ~s[**aaaa~**]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<p><strong>aaaa~</strong></p>]
# strikethrought
code = ~s[<del>aaaa~</del>]
result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
assert result == ~s[<p><del>aaaa~</del></p>]
end
end

117
test/pleroma/web/common_api/utils_test.exs
View file

@ -168,6 +168,123 @@ test "works for text/markdown with mentions" do
end
end
describe "format_input/3 with markdown" do
test "Paragraph" do
code = ~s[Hello\n\nWorld!]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == "<p>Hello</p><p>World!</p>"
end
test "links" do
code = "https://en.wikipedia.org/wiki/Animal_Crossing_(video_game)"
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
code = "https://github.com/pragdave/earmark/"
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
end
test "link with local mention" do
insert(:user, %{nickname: "lain"})
code = "https://example.com/@lain"
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
end
test "local mentions" do
mario = insert(:user, %{nickname: "mario"})
luigi = insert(:user, %{nickname: "luigi"})
code = "@mario @luigi yo what's up?"
{result, _, []} = Utils.format_input(code, "text/markdown")
assert result ==
~s[<p><span class="h-card"><a class="u-url mention" data-user="#{mario.id}" href="#{
mario.ap_id
}" rel="ugc">@<span>mario</span></a></span> <span class="h-card"><a class="u-url mention" data-user="#{
luigi.id
}" href="#{luigi.ap_id}" rel="ugc">@<span>luigi</span></a></span> yo what’s up?</p>]
end
test "remote mentions" do
mario = insert(:user, %{nickname: "mario@mushroom.world", local: false})
luigi = insert(:user, %{nickname: "luigi@mushroom.world", local: false})
code = "@mario@mushroom.world @luigi@mushroom.world yo what's up?"
{result, _, []} = Utils.format_input(code, "text/markdown")
assert result ==
~s[<p><span class="h-card"><a class="u-url mention" data-user="#{mario.id}" href="#{
mario.ap_id
}" rel="ugc">@<span>mario</span></a></span> <span class="h-card"><a class="u-url mention" data-user="#{
luigi.id
}" href="#{luigi.ap_id}" rel="ugc">@<span>luigi</span></a></span> yo what’s up?</p>]
end
test "raw HTML" do
code = ~s[<a href="http://example.org/">OwO</a><!-- what's this?-->]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<a href="http://example.org/">OwO</a>]
end
test "rulers" do
code = ~s[before\n\n-----\n\nafter]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == "<p>before</p><hr/><p>after</p>"
end
test "blockquote" do
code = ~s[> whoms't are you quoting?]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == "<blockquote><p>whoms’t are you quoting?</p></blockquote>"
end
test "code" do
code = ~s[`mix`]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><code class="inline">mix</code></p>]
code = ~s[``mix``]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><code class="inline">mix</code></p>]
code = ~s[```\nputs "Hello World"\n```]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<pre><code>puts &quot;Hello World&quot;</code></pre>]
code = ~s[ <div>\n </div>]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<pre><code>&lt;div&gt;\n&lt;/div&gt;</code></pre>]
end
test "lists" do
code = ~s[- one\n- two\n- three\n- four]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == "<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>"
code = ~s[1. one\n2. two\n3. three\n4. four\n]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == "<ol><li>one</li><li>two</li><li>three</li><li>four</li></ol>"
end
test "delegated renderers" do
code = ~s[*aaaa~*]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><em>aaaa~</em></p>]
code = ~s[**aaaa~**]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><strong>aaaa~</strong></p>]
# strikethrough
code = ~s[~~aaaa~~~]
{result, [], []} = Utils.format_input(code, "text/markdown")
assert result == ~s[<p><del>aaaa</del>~</p>]
end
end
describe "context_to_conversation_id" do
test "creates a mapping object" do
conversation_id = Utils.context_to_conversation_id("random context")

2
test/pleroma/web/common_api_test.exs
View file

@ -597,7 +597,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do
object = Object.normalize(activity, fetch: false)
assert object.data["content"] == "<p><b>2hu</b></p>alert(&#39;xss&#39;)"
assert object.data["content"] == "<p><b>2hu</b></p>"
assert object.data["source"] == post
end