mirrors/akkoma
1
0
Fork 1
mirror of https://akkoma.dev/AkkomaGang/akkoma.git synced 2024-11-17 18:49:15 +00:00
Code Issues Wiki Activity

Merge branch 'media-preview-proxy-nostream' into 'develop'

Browse source 
Media preview proxy

See merge request pleroma/pleroma!3001
This commit is contained in:
feld 2020-09-18 18:38:20 +00:00
parent 91ddd47e51 a428800405
commit 6c052bd5b6
25 changed files with 982 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -15,6 +15,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
- Users with the `discoverable` field set to false will not show up in searches.
- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
### Added
- Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details).
### Removed
- **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation).

14
config/config.exs
View file

@ -434,6 +434,8 @@
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
# Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1
max_read_duration: 30_000,
http: [
follow_redirect: true,
pool: :media
@ -448,6 +450,14 @@
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
# Note: media preview proxy depends on media proxy to be enabled
config :pleroma, :media_preview_proxy,
enabled: false,
thumbnail_max_width: 600,
thumbnail_max_height: 600,
image_quality: 85,
min_content_length: 100 * 1024
config :pleroma, :chat, enabled: true
config :phoenix, :format_encoders, json: Jason
@ -753,8 +763,8 @@
],
media: [
size: 50,
max_waiting: 10,
recv_timeout: 10_000
max_waiting: 20,
recv_timeout: 15_000
],
upload: [
size: 25,

43
config/description.exs
View file

@ -1887,6 +1887,7 @@
suggestions: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
max_read_duration: 30_000,
http: [
follow_redirect: true,
pool: :media
@ -1907,6 +1908,11 @@
"Limits the content length to be approximately the " <>
"specified length. It is validated with the `content-length` header and also verified when proxying."
},
%{
key: :max_read_duration,
type: :integer,
description: "Timeout (in milliseconds) of GET request to remote URI."
},
%{
key: :http,
label: "HTTP",
@ -1953,6 +1959,43 @@
}
]
},
%{
group: :pleroma,
key: :media_preview_proxy,
type: :group,
description: "Media preview proxy",
children: [
%{
key: :enabled,
type: :boolean,
description:
"Enables proxying of remote media preview to the instance's proxy. Requires enabled media proxy."
},
%{
key: :thumbnail_max_width,
type: :integer,
description:
"Max width of preview thumbnail for images (video preview always has original dimensions)."
},
%{
key: :thumbnail_max_height,
type: :integer,
description:
"Max height of preview thumbnail for images (video preview always has original dimensions)."
},
%{
key: :image_quality,
type: :integer,
description: "Quality of the output. Ranges from 0 (min quality) to 100 (max quality)."
},
%{
key: :min_content_length,
type: :integer,
description:
"Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing."
}
]
},
%{
group: :pleroma,
key: Pleroma.Web.MediaProxy.Invalidation.Http,

8
docs/configuration/cheatsheet.md
View file

@ -324,6 +324,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
* `enabled`: Enables purge cache
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
## :media_preview_proxy
* `enabled`: Enables proxying of remote media preview to the instance’s proxy. Requires enabled media proxy (`media_proxy/enabled`).
* `thumbnail_max_width`: Max width of preview thumbnail for images (video preview always has original dimensions).
* `thumbnail_max_height`: Max height of preview thumbnail for images (video preview always has original dimensions).
* `image_quality`: Quality of the output. Ranges from 0 (min quality) to 100 (max quality).
* `min_content_length`: Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing.
### Purge cache strategy
#### Pleroma.Web.MediaProxy.Invalidation.Script

150
lib/pleroma/helpers/media_helper.ex Normal file
View file

@ -0,0 +1,150 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Helpers.MediaHelper do
@moduledoc """
Handles common media-related operations.
"""
alias Pleroma.HTTP
def image_resize(url, options) do
with executable when is_binary(executable) <- System.find_executable("convert"),
{:ok, args} <- prepare_image_resize_args(options),
{:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, fifo_path} <- mkfifo() do
args = List.flatten([fifo_path, args])
run_fifo(fifo_path, env, executable, args)
else
nil -> {:error, {:convert, :command_not_found}}
{:error, _} = error -> error
end
end
defp prepare_image_resize_args(
%{max_width: max_width, max_height: max_height, format: "png"} = options
) do
quality = options[:quality] || 85
resize = Enum.join([max_width, "x", max_height, ">"])
args = [
"-resize",
resize,
"-quality",
to_string(quality),
"png:-"
]
{:ok, args}
end
defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
quality = options[:quality] || 85
resize = Enum.join([max_width, "x", max_height, ">"])
args = [
"-interlace",
"Plane",
"-resize",
resize,
"-quality",
to_string(quality),
"jpg:-"
]
{:ok, args}
end
defp prepare_image_resize_args(_), do: {:error, :missing_options}
# Note: video thumbnail is intentionally not resized (always has original dimensions)
def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
{:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, fifo_path} <- mkfifo(),
args = [
"-y",
"-i",
fifo_path,
"-vframes",
"1",
"-f",
"mjpeg",
"-loglevel",
"error",
"-"
] do
run_fifo(fifo_path, env, executable, args)
else
nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error
end
end
defp run_fifo(fifo_path, env, executable, args) do
pid =
Port.open({:spawn_executable, executable}, [
:use_stdio,
:stream,
:exit_status,
:binary,
args: args
])
fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
fix = Pleroma.Helpers.QtFastStart.fix(env.body)
true = Port.command(fifo, fix)
:erlang.port_close(fifo)
loop_recv(pid)
after
File.rm(fifo_path)
end
defp mkfifo do
path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}")
case System.cmd("mkfifo", [path]) do
{_, 0} ->
spawn(fifo_guard(path))
{:ok, path}
{_, err} ->
{:error, {:fifo_failed, err}}
end
end
defp fifo_guard(path) do
pid = self()
fn ->
ref = Process.monitor(pid)
receive do
{:DOWN, ^ref, :process, ^pid, _} ->
File.rm(path)
end
end
end
defp loop_recv(pid) do
loop_recv(pid, <<>>)
end
defp loop_recv(pid, acc) do
receive do
{^pid, {:data, data}} ->
loop_recv(pid, acc <> data)
{^pid, {:exit_status, 0}} ->
{:ok, acc}
{^pid, {:exit_status, status}} ->
{:error, status}
after
5000 ->
:erlang.port_close(pid)
{:error, :timeout}
end
end
end

131
lib/pleroma/helpers/qt_fast_start.ex Normal file
View file

@ -0,0 +1,131 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Helpers.QtFastStart do
@moduledoc """
(WIP) Converts a "slow start" (data before metadatas) mov/mp4 file to a "fast start" one (metadatas before data).
"""
# TODO: Cleanup and optimizations
# Inspirations: https://www.ffmpeg.org/doxygen/3.4/qt-faststart_8c_source.html
# https://github.com/danielgtaylor/qtfaststart/blob/master/qtfaststart/processor.py
# ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
# Paracetamol
def fix(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>> = binary) do
index = fix(binary, 0, nil, nil, [])
case index do
:abort -> binary
[{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
[{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
_ -> binary
end
end
def fix(binary) do
binary
end
# MOOV have been seen before MDAT- abort
defp fix(<<_::bits>>, _, true, false, _) do
:abort
end
defp fix(
<<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
pos,
got_moov,
got_mdat,
acc
) do
full_size = (size - 8) * 8
<<data::bits-size(full_size), rest::bits>> = rest
acc = [
{fourcc, pos, pos + size, size,
<<size::integer-big-size(32), fourcc::bits-size(32), data::bits>>}
| acc
]
fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc)
end
defp fix(<<>>, _pos, _, _, acc) do
:lists.reverse(acc)
end
defp faststart(index) do
{{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
# Skip re-writing the free fourcc as it's kind of useless.
# Why stream useless bytes when you can do without?
{free_size, index} =
case List.keytake(index, "free", 0) do
{{_, _, _, size, _}, index} -> {size, index}
_ -> {0, index}
end
{{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
offset = -free_size + moov_size
rest = for {_, _, _, _, data} <- index, do: data, into: []
<<moov_head::bits-size(64), moov_data::bits>> = moov
[ftyp, moov_head, fix_moov(moov_data, offset, []), rest]
end
defp fix_moov(
<<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
offset,
acc
) do
full_size = (size - 8) * 8
<<data::bits-size(full_size), rest::bits>> = rest
data =
cond do
fourcc in ["trak", "mdia", "minf", "stbl"] ->
# Theses contains sto or co64 part
[<<size::integer-big-size(32), fourcc::bits-size(32)>>, fix_moov(data, offset, [])]
fourcc in ["stco", "co64"] ->
# fix the damn thing
<<version::integer-big-size(32), count::integer-big-size(32), rest::bits>> = data
entry_size =
case fourcc do
"stco" -> 32
"co64" -> 64
end
[
<<size::integer-big-size(32), fourcc::bits-size(32), version::integer-big-size(32),
count::integer-big-size(32)>>,
rewrite_entries(entry_size, offset, rest, [])
]
true ->
[<<size::integer-big-size(32), fourcc::bits-size(32)>>, data]
end
acc = [acc | data]
fix_moov(rest, offset, acc)
end
defp fix_moov(<<>>, _, acc), do: acc
for size <- [32, 64] do
defp rewrite_entries(
unquote(size),
offset,
<<pos::integer-big-size(unquote(size)), rest::bits>>,
acc
) do
rewrite_entries(unquote(size), offset, rest, [
acc | <<pos + offset::integer-big-size(unquote(size))>>
])
end
end
defp rewrite_entries(_, _, <<>>, acc), do: acc
end

14
lib/pleroma/helpers/uri_helper.ex
View file

@ -3,18 +3,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Helpers.UriHelper do
def append_uri_params(uri, appended_params) do
def modify_uri_params(uri, overridden_params, deleted_params \\ []) do
uri = URI.parse(uri)
appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
existing_params = URI.query_decoder(uri.query || "") |> Map.new()
overridden_params = Map.new(overridden_params, fn {k, v} -> {to_string(k), v} end)
deleted_params = Enum.map(deleted_params, &to_string/1)
updated_params =
for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
existing_params
|> Map.merge(overridden_params)
|> Map.drop(deleted_params)
uri
|> Map.put(:query, URI.encode_query(updated_params))
|> URI.to_string()
|> String.replace_suffix("?", "")
end
def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])

4
lib/pleroma/instances/instance.ex
View file

@ -156,9 +156,7 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
adapter: [pool: :media]
),
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
{:parse,
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},

5
lib/pleroma/reverse_proxy/reverse_proxy.ex
View file

@ -17,6 +17,9 @@ defmodule Pleroma.ReverseProxy do
@failed_request_ttl :timer.seconds(60)
@methods ~w(GET HEAD)
def max_read_duration_default, do: @max_read_duration
def default_cache_control_header, do: @default_cache_control_header
@moduledoc """
A reverse proxy.
@ -391,6 +394,8 @@ defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and
defp body_size_constraint(_, _), do: :ok
defp check_read_duration(nil = _duration, max), do: check_read_duration(@max_read_duration, max)
defp check_read_duration(duration, max)
when is_integer(duration) and is_integer(max) and max > 0 do
if duration > max do

14
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -12,17 +12,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@options [
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}")
# Fetching only proxiable resources
if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
# If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
prefetch_url = MediaProxy.preview_url(url)
url
|> MediaProxy.url()
|> HTTP.get([], @options)
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
HTTP.get(prefetch_url, [], @adapter_options)
end
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

10
lib/pleroma/web/mastodon_api/views/account_view.ex
View file

@ -181,8 +181,10 @@ defp do_render("show.json", %{user: user} = opts) do
user = User.sanitize_html(user, User.html_filter_policy(opts[:for]))
display_name = user.name || user.nickname
image = User.avatar_url(user) |> MediaProxy.url()
avatar = User.avatar_url(user) |> MediaProxy.url()
avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true)
header = User.banner_url(user) |> MediaProxy.url()
header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true)
following_count =
if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
@ -247,10 +249,10 @@ defp do_render("show.json", %{user: user} = opts) do
statuses_count: user.note_count,
note: user.bio,
url: user.uri || user.ap_id,
avatar: image,
avatar_static: image,
avatar: avatar,
avatar_static: avatar_static,
header: header,
header_static: header,
header_static: header_static,
emojis: emojis,
fields: user.fields,
bot: bot,

3
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -415,6 +415,7 @@ def render("attachment.json", %{attachment: attachment}) do
[attachment_url | _] = attachment["url"]
media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
href = attachment_url["href"] |> MediaProxy.url()
href_preview = attachment_url["href"] |> MediaProxy.preview_url()
type =
cond do
@ -430,7 +431,7 @@ def render("attachment.json", %{attachment: attachment}) do
id: to_string(attachment["id"] || hash_id),
url: href,
remote_url: href,
preview_url: href,
preview_url: href_preview,
text_url: href,
type: type,
description: attachment["name"],

4
lib/pleroma/web/media_proxy/invalidation.ex
View file

@ -33,6 +33,8 @@ defp do_purge(urls) do
def prepare_urls(urls) do
urls
|> List.wrap()
|> Enum.map(&MediaProxy.url/1)
|> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end)
|> List.flatten()
|> Enum.uniq()
end
end

87
lib/pleroma/web/media_proxy/media_proxy.ex
View file

@ -4,6 +4,7 @@
defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Config
alias Pleroma.Helpers.UriHelper
alias Pleroma.Upload
alias Pleroma.Web
alias Pleroma.Web.MediaProxy.Invalidation
@ -40,27 +41,35 @@ def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
if disabled?() or not url_proxiable?(url) do
url
else
if enabled?() and url_proxiable?(url) do
encode_url(url)
else
url
end
end
@spec url_proxiable?(String.t()) :: boolean()
def url_proxiable?(url) do
if local?(url) or whitelisted?(url) do
false
not local?(url) and not whitelisted?(url)
end
def preview_url(url, preview_params \\ []) do
if preview_enabled?() do
encode_preview_url(url, preview_params)
else
true
url(url)
end
end
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
def enabled?, do: Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
# Note: media proxy must be enabled for media preview proxy in order to load all
# non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
defp whitelisted?(url) do
def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
def whitelisted?(url) do
%{host: domain} = URI.parse(url)
mediaproxy_whitelist_domains =
@ -85,17 +94,29 @@ defp maybe_get_domain_from_url("http" <> _ = url) do
defp maybe_get_domain_from_url(domain), do: domain
def encode_url(url) do
defp base64_sig64(url) do
base64 = Base.url_encode64(url, @base64_opts)
sig64 =
base64
|> signed_url
|> signed_url()
|> Base.url_encode64(@base64_opts)
{base64, sig64}
end
def encode_url(url) do
{base64, sig64} = base64_sig64(url)
build_url(sig64, base64, filename(url))
end
def encode_preview_url(url, preview_params \\ []) do
{base64, sig64} = base64_sig64(url)
build_preview_url(sig64, base64, filename(url), preview_params)
end
def decode_url(sig, url) do
with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
signature when signature == sig <- signed_url(url) do
@ -113,10 +134,14 @@ def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
def base_url do
Config.get([:media_proxy, :base_url], Web.base_url())
end
defp proxy_url(path, sig_base64, url_base64, filename) do
[
Config.get([:media_proxy, :base_url], Web.base_url()),
"proxy",
base_url(),
path,
sig_base64,
url_base64,
filename
@ -124,4 +149,38 @@ def build_url(sig_base64, url_base64, filename \\ nil) do
|> Enum.filter(& &1)
|> Path.join()
end
def build_url(sig_base64, url_base64, filename \\ nil) do
proxy_url("proxy", sig_base64, url_base64, filename)
end
def build_preview_url(sig_base64, url_base64, filename \\ nil, preview_params \\ []) do
uri = proxy_url("proxy/preview", sig_base64, url_base64, filename)
UriHelper.modify_uri_params(uri, preview_params)
end
def verify_request_path_and_url(
%Plug.Conn{params: %{"filename" => _}, request_path: request_path},
url
) do
verify_request_path_and_url(request_path, url)
end
def verify_request_path_and_url(request_path, url) when is_binary(request_path) do
filename = filename(url)
if filename && not basename_matches?(request_path, filename) do
{:wrong_filename, filename}
else
:ok
end
end
def verify_request_path_and_url(_, _), do: :ok
defp basename_matches?(path, filename) do
basename = Path.basename(path)
basename == filename or URI.decode(basename) == filename or URI.encode(basename) == filename
end
end

197
lib/pleroma/web/media_proxy/media_proxy_controller.ex
View file

@ -5,44 +5,201 @@
defmodule Pleroma.Web.MediaProxy.MediaProxyController do
use Pleroma.Web, :controller
alias Pleroma.Config
alias Pleroma.Helpers.MediaHelper
alias Pleroma.Helpers.UriHelper
alias Pleroma.ReverseProxy
alias Pleroma.Web.MediaProxy
alias Plug.Conn
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
def remote(conn, %{"sig" => sig64, "url" => url64}) do
with {_, true} <- {:enabled, MediaProxy.enabled?()},
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
{_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
:ok <- MediaProxy.verify_request_path_and_url(conn, url) do
ReverseProxy.call(conn, url, media_proxy_opts())
else
error when error in [false, {:in_banned_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:enabled, false} ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:in_banned_urls, true} ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
end
end
def filename_matches(%{"filename" => _} = _, path, url) do
filename = MediaProxy.filename(url)
if filename && does_not_match(path, filename) do
{:wrong_filename, filename}
def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
:ok <- MediaProxy.verify_request_path_and_url(conn, url) do
handle_preview(conn, url)
else
:ok
{:enabled, false} ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_preview_url(sig64, url64, filename))
end
end
def filename_matches(_, _, _), do: :ok
defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url)
defp does_not_match(path, filename) do
basename = Path.basename(path)
basename != filename and URI.decode(basename) != filename and URI.encode(basename) != filename
with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request("head", media_proxy_url, [], [], pool: :media) do
content_type = Tesla.get_header(head_response, "content-type")
content_length = Tesla.get_header(head_response, "content-length")
content_length = content_length && String.to_integer(content_length)
static = conn.params["static"] in ["true", true]
cond do
static and content_type == "image/gif" ->
handle_jpeg_preview(conn, media_proxy_url)
static ->
drop_static_param_and_redirect(conn)
content_type == "image/gif" ->
redirect(conn, external: media_proxy_url)
min_content_length_for_preview() > 0 and content_length > 0 and
content_length < min_content_length_for_preview() ->
redirect(conn, external: media_proxy_url)
true ->
handle_preview(content_type, conn, media_proxy_url)
end
else
# If HEAD failed, redirecting to media proxy URI doesn't make much sense; returning an error
{_, %{status: status}} ->
send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
{:error, :recv_response_timeout} ->
send_resp(conn, :failed_dependency, "HEAD request timeout.")
_ ->
send_resp(conn, :failed_dependency, "Can't fetch HTTP headers.")
end
end
defp handle_preview("image/png" <> _ = _content_type, conn, media_proxy_url) do
handle_png_preview(conn, media_proxy_url)
end
defp handle_preview("image/" <> _ = _content_type, conn, media_proxy_url) do
handle_jpeg_preview(conn, media_proxy_url)
end
defp handle_preview("video/" <> _ = _content_type, conn, media_proxy_url) do
handle_video_preview(conn, media_proxy_url)
end
defp handle_preview(_unsupported_content_type, conn, media_proxy_url) do
fallback_on_preview_error(conn, media_proxy_url)
end
defp handle_png_preview(conn, media_proxy_url) do
quality = Config.get!([:media_preview_proxy, :image_quality])
{thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
with {:ok, thumbnail_binary} <-
MediaHelper.image_resize(
media_proxy_url,
%{
max_width: thumbnail_max_width,
max_height: thumbnail_max_height,
quality: quality,
format: "png"
}
) do
conn
|> put_preview_response_headers(["image/png", "preview.png"])
|> send_resp(200, thumbnail_binary)
else
_ ->
fallback_on_preview_error(conn, media_proxy_url)
end
end
defp handle_jpeg_preview(conn, media_proxy_url) do
quality = Config.get!([:media_preview_proxy, :image_quality])
{thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
with {:ok, thumbnail_binary} <-
MediaHelper.image_resize(
media_proxy_url,
%{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
) do
conn
|> put_preview_response_headers()
|> send_resp(200, thumbnail_binary)
else
_ ->
fallback_on_preview_error(conn, media_proxy_url)
end
end
defp handle_video_preview(conn, media_proxy_url) do
with {:ok, thumbnail_binary} <-
MediaHelper.video_framegrab(media_proxy_url) do
conn
|> put_preview_response_headers()
|> send_resp(200, thumbnail_binary)
else
_ ->
fallback_on_preview_error(conn, media_proxy_url)
end
end
defp drop_static_param_and_redirect(conn) do
uri_without_static_param =
conn
|> current_url()
|> UriHelper.modify_uri_params(%{}, ["static"])
redirect(conn, external: uri_without_static_param)
end
defp fallback_on_preview_error(conn, media_proxy_url) do
redirect(conn, external: media_proxy_url)
end
defp put_preview_response_headers(
conn,
[content_type, filename] = _content_info \\ ["image/jpeg", "preview.jpg"]
) do
conn
|> put_resp_header("content-type", content_type)
|> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
|> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
end
defp thumbnail_max_dimensions do
config = media_preview_proxy_config()
thumbnail_max_width = Keyword.fetch!(config, :thumbnail_max_width)
thumbnail_max_height = Keyword.fetch!(config, :thumbnail_max_height)
{thumbnail_max_width, thumbnail_max_height}
end
defp min_content_length_for_preview do
Keyword.get(media_preview_proxy_config(), :min_content_length, 0)
end
defp media_preview_proxy_config do
Config.get!([:media_preview_proxy])
end
defp media_proxy_opts do
Config.get([:media_proxy, :proxy_opts], [])
end
end

2
lib/pleroma/web/metadata/utils.ex
View file

@ -38,7 +38,7 @@ def scrub_html(content) when is_binary(content) do
def scrub_html(content), do: content
def attachment_url(url) do
MediaProxy.url(url)
MediaProxy.preview_url(url)
end
def user_name_string(user) do

4
lib/pleroma/web/oauth/oauth_controller.ex
View file

@ -119,7 +119,7 @@ defp handle_existing_authorization(
redirect_uri = redirect_uri(conn, redirect_uri)
url_params = %{access_token: token.token}
url_params = Maps.put_if_present(url_params, :state, params["state"])
url = UriHelper.append_uri_params(redirect_uri, url_params)
url = UriHelper.modify_uri_params(redirect_uri, url_params)
redirect(conn, external: url)
else
conn
@ -161,7 +161,7 @@ def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{
redirect_uri = redirect_uri(conn, redirect_uri)
url_params = %{code: auth.token}
url_params = Maps.put_if_present(url_params, :state, auth_attrs["state"])
url = UriHelper.append_uri_params(redirect_uri, url_params)
url = UriHelper.modify_uri_params(redirect_uri, url_params)
redirect(conn, external: url)
else
conn

2
lib/pleroma/web/router.ex
View file

@ -679,6 +679,8 @@ defmodule Pleroma.Web.Router do
end
scope "/proxy/", Pleroma.Web.MediaProxy do
get("/preview/:sig/:url", MediaProxyController, :preview)
get("/preview/:sig/:url/:filename", MediaProxyController, :preview)
get("/:sig/:url", MediaProxyController, :remote)
get("/:sig/:url/:filename", MediaProxyController, :remote)
end

2
mix.lock
View file

@ -31,6 +31,7 @@
"ecto": {:hex, :ecto, "3.4.5", "2bcd262f57b2c888b0bd7f7a28c8a48aa11dc1a2c6a858e45dd8f8426d504265", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c6d1d4d524559e9b7a062f0498e2c206122552d63eacff0a6567ffe7a8e8691"},
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
"ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
"eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
"elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"},
"esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"},
"eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"},
@ -80,6 +81,7 @@
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
"oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"},
"open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},
"p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
"pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
"phoenix": {:hex, :phoenix, "1.4.17", "1b1bd4cff7cfc87c94deaa7d60dd8c22e04368ab95499483c50640ef3bd838d8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a8e5d7a3d76d452bb5fb86e8b7bd115f737e4f8efe202a463d4aeb4a5809611"},

BIN
test/fixtures/image.gif vendored Executable file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 978 KiB

BIN
test/fixtures/image.png vendored Executable file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 102 KiB

2
test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs
View file

@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do
}
}
setup do: clear_config([:media_proxy, :enabled], true)
test "it prefetches media proxy URIs" do
with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
MediaProxyWarmingPolicy.filter(@message)

34
test/web/mastodon_api/views/account_view_test.exs
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
@ -540,8 +541,9 @@ test "shows non-zero when historical unapproved requests are present" do
end
end
test "uses mediaproxy urls when it's enabled" do
test "uses mediaproxy urls when it's enabled (regardless of media preview proxy state)" do
clear_config([:media_proxy, :enabled], true)
clear_config([:media_preview_proxy, :enabled])
user =
insert(:user,
@ -550,20 +552,24 @@ test "uses mediaproxy urls when it's enabled" do
emoji: %{"joker_smile" => "https://evil.website/society.png"}
)
AccountView.render("show.json", %{user: user, skip_visibility_check: true})
|> Enum.all?(fn
{key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
String.starts_with?(url, Pleroma.Web.base_url())
with media_preview_enabled <- [false, true] do
Config.put([:media_preview_proxy, :enabled], media_preview_enabled)
{:emojis, emojis} ->
Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
String.starts_with?(url, Pleroma.Web.base_url()) &&
String.starts_with?(static_url, Pleroma.Web.base_url())
end)
AccountView.render("show.json", %{user: user, skip_visibility_check: true})
|> Enum.all?(fn
{key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
String.starts_with?(url, Pleroma.Web.base_url())
_ ->
true
end)
|> assert()
{:emojis, emojis} ->
Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
String.starts_with?(url, Pleroma.Web.base_url()) &&
String.starts_with?(static_url, Pleroma.Web.base_url())
end)
_ ->
true
end)
|> assert()
end
end
end

302
test/web/media_proxy/media_proxy_controller_test.exs
View file

@ -8,34 +8,34 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
import Mock
alias Pleroma.Web.MediaProxy
alias Pleroma.Web.MediaProxy.MediaProxyController
alias Plug.Conn
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
clear_config([:media_proxy, :enabled], false)
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeeee")
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeee/fff")
end
describe "" do
describe "Media Proxy" do
setup do
clear_config([:media_proxy, :enabled], true)
clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
[url: MediaProxy.encode_url("https://google.fn/test.png")]
end
test "it returns 404 when disabled", %{conn: conn} do
clear_config([:media_proxy, :enabled], false)
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeeee")
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeee/fff")
end
test "it returns 403 for invalid signature", %{conn: conn, url: url} do
Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
%{path: path} = URI.parse(url)
@ -56,7 +56,7 @@ test "it returns 403 for invalid signature", %{conn: conn, url: url} do
} = get(conn, "/proxy/hhgfh/eeee/fff")
end
test "redirects on valid url when filename is invalidated", %{conn: conn, url: url} do
test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
invalid_url = String.replace(url, "test.png", "test-file.png")
response = get(conn, invalid_url)
assert response.status == 302
@ -80,42 +80,248 @@ test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url}
end
end
describe "filename_matches/3" do
test "preserves the encoded or decoded path" do
assert MediaProxyController.filename_matches(
%{"filename" => "/Hello world.jpg"},
"/Hello world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
describe "Media Preview Proxy" do
setup do
clear_config([:media_proxy, :enabled], true)
clear_config([:media_preview_proxy, :enabled], true)
clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
assert MediaProxyController.filename_matches(
%{"filename" => "/Hello%20world.jpg"},
"/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
original_url = "https://google.fn/test.png"
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jp"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jp",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
[
url: MediaProxy.encode_preview_url(original_url),
media_proxy_url: MediaProxy.encode_url(original_url)
]
end
test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
# conn.request_path will return encoded url
request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
test "returns 404 when media proxy is disabled", %{conn: conn} do
clear_config([:media_proxy, :enabled], false)
assert MediaProxyController.filename_matches(
true,
request_path,
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
) == :ok
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/preview/hhgfh/eeeee")
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/preview/hhgfh/fff")
end
test "returns 404 when disabled", %{conn: conn} do
clear_config([:media_preview_proxy, :enabled], false)
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/preview/hhgfh/eeeee")
assert %Conn{
status: 404,
resp_body: "Not Found"
} = get(conn, "/proxy/preview/hhgfh/fff")
end
test "it returns 403 for invalid signature", %{conn: conn, url: url} do
Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
%{path: path} = URI.parse(url)
assert %Conn{
status: 403,
resp_body: "Forbidden"
} = get(conn, path)
assert %Conn{
status: 403,
resp_body: "Forbidden"
} = get(conn, "/proxy/preview/hhgfh/eeee")
assert %Conn{
status: 403,
resp_body: "Forbidden"
} = get(conn, "/proxy/preview/hhgfh/eeee/fff")
end
test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
invalid_url = String.replace(url, "test.png", "test-file.png")
response = get(conn, invalid_url)
assert response.status == 302
assert redirected_to(response) == url
end
test "responds with 424 Failed Dependency if HEAD request to media proxy fails", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 500, body: ""}
end)
response = get(conn, url)
assert response.status == 424
assert response.resp_body == "Can't fetch HTTP headers (HTTP 500)."
end
test "redirects to media proxy URI on unsupported content type", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "application/pdf"}]}
end)
response = get(conn, url)
assert response.status == 302
assert redirected_to(response) == media_proxy_url
end
test "with `static=true` and GIF image preview requested, responds with JPEG image", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
# Setting a high :min_content_length to ensure this scenario is not affected by its logic
clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000)
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{
status: 200,
body: "",
headers: [{"content-type", "image/gif"}, {"content-length", "1001718"}]
}
%{method: :get, url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.gif")}
end)
response = get(conn, url <> "?static=true")
assert response.status == 200
assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
assert response.resp_body != ""
end
test "with GIF image preview requested and no `static` param, redirects to media proxy URI",
%{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/gif"}]}
end)
response = get(conn, url)
assert response.status == 302
assert redirected_to(response) == media_proxy_url
end
test "with `static` param and non-GIF image preview requested, " <>
"redirects to media preview proxy URI without `static` param",
%{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
end)
response = get(conn, url <> "?static=true")
assert response.status == 302
assert redirected_to(response) == url
end
test "with :min_content_length setting not matched by Content-Length header, " <>
"redirects to media proxy URI",
%{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
clear_config([:media_preview_proxy, :min_content_length], 100_000)
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{
status: 200,
body: "",
headers: [{"content-type", "image/gif"}, {"content-length", "5000"}]
}
end)
response = get(conn, url)
assert response.status == 302
assert redirected_to(response) == media_proxy_url
end
test "thumbnails PNG images into PNG", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]}
%{method: :get, url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.png")}
end)
response = get(conn, url)
assert response.status == 200
assert Conn.get_resp_header(response, "content-type") == ["image/png"]
assert response.resp_body != ""
end
test "thumbnails JPEG images into JPEG", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
%{method: :get, url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
end)
response = get(conn, url)
assert response.status == 200
assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
assert response.resp_body != ""
end
test "redirects to media proxy URI in case of thumbnailing error", %{
conn: conn,
url: url,
media_proxy_url: media_proxy_url
} do
Tesla.Mock.mock(fn
%{method: "head", url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
%{method: :get, url: ^media_proxy_url} ->
%Tesla.Env{status: 200, body: "<html><body>error</body></html>"}
end)
response = get(conn, url)
assert response.status == 302
assert redirected_to(response) == media_proxy_url
end
end
end

73
test/web/media_proxy/media_proxy_test.exs
View file

@ -6,9 +6,16 @@ defmodule Pleroma.Web.MediaProxyTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.Config
alias Pleroma.Web.Endpoint
alias Pleroma.Web.MediaProxy
defp decode_result(encoded) do
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
{:ok, decoded} = MediaProxy.decode_url(sig, base64)
decoded
end
describe "when enabled" do
setup do: clear_config([:media_proxy, :enabled], true)
@ -35,7 +42,7 @@ test "encodes and decodes URL" do
assert String.starts_with?(
encoded,
Pleroma.Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
)
assert String.ends_with?(encoded, "/logo.png")
@ -75,6 +82,64 @@ test "validates signature" do
assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
end
def test_verify_request_path_and_url(request_path, url, expected_result) do
assert MediaProxy.verify_request_path_and_url(request_path, url) == expected_result
assert MediaProxy.verify_request_path_and_url(
%Plug.Conn{
params: %{"filename" => Path.basename(request_path)},
request_path: request_path
},
url
) == expected_result
end
test "if first arg of `verify_request_path_and_url/2` is a Plug.Conn without \"filename\" " <>
"parameter, `verify_request_path_and_url/2` returns :ok " do
assert MediaProxy.verify_request_path_and_url(
%Plug.Conn{params: %{}, request_path: "/some/path"},
"https://instance.com/file.jpg"
) == :ok
assert MediaProxy.verify_request_path_and_url(
%Plug.Conn{params: %{}, request_path: "/path/to/file.jpg"},
"https://instance.com/file.jpg"
) == :ok
end
test "`verify_request_path_and_url/2` preserves the encoded or decoded path" do
test_verify_request_path_and_url(
"/Hello world.jpg",
"http://pleroma.social/Hello world.jpg",
:ok
)
test_verify_request_path_and_url(
"/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg",
:ok
)
test_verify_request_path_and_url(
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
:ok
)
test_verify_request_path_and_url(
# Note: `conn.request_path` returns encoded url
"/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg",
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg",
:ok
)
test_verify_request_path_and_url(
"/my%2Flong%2Furl%2F2019%2F07%2FS",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
{:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
)
end
test "uses the configured base_url" do
base_url = "https://cache.pleroma.social"
clear_config([:media_proxy, :base_url], base_url)
@ -124,12 +189,6 @@ test "does not encode remote urls" do
end
end
defp decode_result(encoded) do
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
{:ok, decoded} = MediaProxy.decode_url(sig, base64)
decoded
end
describe "whitelist" do
setup do: clear_config([:media_proxy, :enabled], true)