erin/akkoma
1
0
Fork 0
forked from mirrors/akkoma
Code Activity

Merge branch 'deprecate-public_endpoint' into 'develop'

Browse source 
Deprecate Uploaders.S3, :public_endpoint

See merge request pleroma/pleroma!3251
This commit is contained in:
feld 2021-01-20 22:48:48 +00:00
parent 2905df841b 086100e3b7
commit 2926713fe5
18 changed files with 226 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
- Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators.
- Admin API: Reports now ordered by newest
- Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders.
### Added

15
config/config.exs
View file

@ -64,14 +64,23 @@
link_name: false,
proxy_remote: false,
filename_display_max_length: 30,
default_description: nil
default_description: nil,
base_url: nil
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
streaming_enabled: true,
public_endpoint: "https://s3.amazonaws.com"
bucket_namespace: nil,
truncated_namespace: nil,
streaming_enabled: true
config :ex_aws, :s3,
# host: "s3.wasabisys.com", # required if not Amazon AWS
access_key_id: nil,
secret_access_key: nil,
# region: "us-east-1", # may be required for Amazon AWS
scheme: "https://"
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],

8
config/description.exs
View file

@ -149,18 +149,12 @@
description: "S3 bucket namespace",
suggestions: ["pleroma"]
},
%{
key: :public_endpoint,
type: :string,
description: "S3 endpoint",
suggestions: ["https://s3.amazonaws.com"]
},
%{
key: :truncated_namespace,
type: :string,
description:
"If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or \"\" etc." <>
" For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in public_endpoint."
" For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in Upload base_url."
},
%{
key: :streaming_enabled,

5
config/test.exs
View file

@ -115,11 +115,6 @@
config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
streaming_enabled: true,
public_endpoint: nil
config :tzdata, :autoupdate, :disabled
config :pleroma, :mrf, policies: []

5
docs/configuration/cheatsheet.md
View file

@ -549,7 +549,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
* `uploader`: Which one of the [uploaders](#uploaders) to use.
* `filters`: List of [upload filters](#upload-filters) to use.
* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to host the media files via another domain or are using a 3rd party S3 provider.
* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
* `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
@ -570,10 +570,7 @@ Don't forget to configure [Ex AWS S3](#ex-aws-s3-settings)
* `bucket`: S3 bucket name.
* `bucket_namespace`: S3 bucket namespace.
* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com")
* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
For example, when using CDN to S3 virtual host format, set "".
At this time, write CNAME to CDN in public_endpoint.
* `streaming_enabled`: Enable streaming uploads, when enabled the file will be sent to the server in chunks as it's being read. This may be unsupported by some providers, try disabling this if you have upload problems.
#### Ex AWS S3 settings

24
lib/pleroma/config/deprecation_warnings.ex
View file

@ -40,7 +40,8 @@ def warn do
:ok <- check_welcome_message_config(),
:ok <- check_gun_pool_options(),
:ok <- check_activity_expiration_config(),
:ok <- check_remote_ip_plug_name() do
:ok <- check_remote_ip_plug_name(),
:ok <- check_uploders_s3_public_endpoint() do
:ok
else
_ ->
@ -193,4 +194,25 @@ def check_remote_ip_plug_name do
warning_preface
)
end
@spec check_uploders_s3_public_endpoint() :: :ok | nil
def check_uploders_s3_public_endpoint do
s3_config = Pleroma.Config.get([Pleroma.Uploaders.S3])
use_old_config = Keyword.has_key?(s3_config, :public_endpoint)
if use_old_config do
Logger.error("""
!!!DEPRECATION WARNING!!!
Your config is using the old setting for controlling the URL of media uploaded to your S3 bucket.\n
Please make the following change at your earliest convenience.\n
\n* `config :pleroma, Pleroma.Uploaders.S3, public_endpoint` is now equal to:
\n* `config :pleroma, Pleroma.Upload, base_url`
""")
:error
else
:ok
end
end
end

24
lib/pleroma/upload.ex
View file

@ -131,12 +131,7 @@ defp get_opts(opts) do
uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])),
filters: Keyword.get(opts, :filters, Pleroma.Config.get([__MODULE__, :filters])),
description: Keyword.get(opts, :description),
base_url:
Keyword.get(
opts,
:base_url,
Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
)
base_url: base_url()
}
end
@ -217,14 +212,7 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
""
end
prefix =
if is_nil(Pleroma.Config.get([__MODULE__, :base_url])) do
"media"
else
""
end
[base_url, prefix, path]
[base_url, path]
|> Path.join()
end
@ -241,13 +229,15 @@ def base_url do
Pleroma.Uploaders.S3 ->
bucket = Config.get([Pleroma.Uploaders.S3, :bucket])
truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace])
namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace])
bucket_with_namespace =
cond do
truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace]) ->
!is_nil(truncated_namespace) ->
truncated_namespace
namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace]) ->
!is_nil(namespace) ->
namespace <> ":" <> bucket
true ->
@ -261,7 +251,7 @@ def base_url do
end
_ ->
public_endpoint || upload_base_url
public_endpoint || upload_base_url || Pleroma.Web.base_url() <> "/media/"
end
end
end

57
priv/repo/migrations/20210113225652_deprecate_public_endpoint.exs Normal file
View file

@ -0,0 +1,57 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Repo.Migrations.DeprecatePublicEndpoint do
use Ecto.Migration
def up do
with %Pleroma.ConfigDB{} = s3_config <-
Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Uploaders.S3}),
%Pleroma.ConfigDB{} = upload_config <-
Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) do
public_endpoint = s3_config.value[:public_endpoint]
if !is_nil(public_endpoint) do
upload_value = upload_config.value |> Keyword.merge(base_url: public_endpoint)
upload_config
|> Ecto.Changeset.change(value: upload_value)
|> Pleroma.Repo.update()
s3_value = s3_config.value |> Keyword.delete(:public_endpoint)
s3_config
|> Ecto.Changeset.change(value: s3_value)
|> Pleroma.Repo.update()
end
else
_ -> :ok
end
end
def down do
with %Pleroma.ConfigDB{} = upload_config <-
Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}),
%Pleroma.ConfigDB{} = s3_config <-
Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Uploaders.S3}) do
base_url = upload_config.value[:base_url]
if !is_nil(base_url) do
s3_value = s3_config.value |> Keyword.merge(public_endpoint: base_url)
s3_config
|> Ecto.Changeset.change(value: s3_value)
|> Pleroma.Repo.update()
upload_value = upload_config.value |> Keyword.delete(:base_url)
upload_config
|> Ecto.Changeset.change(value: upload_value)
|> Pleroma.Repo.update()
end
else
_ -> :ok
end
end
end

10
priv/templates/sample_config.eex
View file

@ -49,12 +49,18 @@ config :pleroma, Pleroma.Uploaders.Local, uploads: "<%= uploads_dir %>"
# sts: true
# Configure S3 support if desired.
# The public S3 endpoint is different depending on region and provider,
# The public S3 endpoint (base_url) is different depending on region and provider,
# consult your S3 provider's documentation for details on what to use.
#
# config :pleroma, Pleroma.Upload,
# uploader: Pleroma.Uploaders.S3,
# base_url: "https://s3.amazonaws.com"
#
# config :pleroma, Pleroma.Uploaders.S3,
# bucket: "some-bucket",
# public_endpoint: "https://s3.amazonaws.com"
# bucket_namespace: "my-namespace",
# truncated_namespace: nil,
# streaming_enabled: true
#
# Configure S3 credentials:
# config :ex_aws, :s3,

9
test/pleroma/config/deprecation_warnings_test.exs
View file

@ -94,6 +94,15 @@ test "check_activity_expiration_config/0" do
end) =~ "Your config is using old namespace for activity expiration configuration."
end
test "check_uploders_s3_public_endpoint/0" do
clear_config(Pleroma.Uploaders.S3, public_endpoint: "https://fake.amazonaws.com/bucket/")
assert capture_log(fn ->
DeprecationWarnings.check_uploders_s3_public_endpoint()
end) =~
"Your config is using the old setting for controlling the URL of media uploaded to your S3 bucket."
end
describe "check_gun_pool_options/0" do
test "await_up_timeout" do
config = Config.get(:connections_pool)

24
test/pleroma/object_test.exs
View file

@ -78,8 +78,8 @@ test "ensures cache is cleared for the object" do
setup do: clear_config([:instance, :cleanup_attachments])
test "Disabled via config" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], false)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([:instance, :cleanup_attachments], false)
file = %Plug.Upload{
content_type: "image/jpeg",
@ -112,8 +112,8 @@ test "Disabled via config" do
end
test "in subdirectories" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], true)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([:instance, :cleanup_attachments], true)
file = %Plug.Upload{
content_type: "image/jpeg",
@ -146,9 +146,9 @@ test "in subdirectories" do
end
test "with dedupe enabled" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
Pleroma.Config.put([:instance, :cleanup_attachments], true)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
clear_config([:instance, :cleanup_attachments], true)
uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
@ -184,8 +184,8 @@ test "with dedupe enabled" do
end
test "with objects that have legacy data.url attribute" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], true)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([:instance, :cleanup_attachments], true)
file = %Plug.Upload{
content_type: "image/jpeg",
@ -220,9 +220,9 @@ test "with objects that have legacy data.url attribute" do
end
test "With custom base_url" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
Pleroma.Config.put([:instance, :cleanup_attachments], true)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
clear_config([:instance, :cleanup_attachments], true)
file = %Plug.Upload{
content_type: "image/jpeg",

60
test/pleroma/repo/migrations/deprecate_public_endpoint_test.exs Normal file
View file

@ -0,0 +1,60 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Repo.Migrations.DeprecatePublicEndpointTest do
use Pleroma.DataCase
import Pleroma.Factory
import Pleroma.Tests.Helpers
alias Pleroma.ConfigDB
setup do: clear_config(Pleroma.Upload)
setup do: clear_config(Pleroma.Uploaders.S3)
setup_all do: require_migration("20210113225652_deprecate_public_endpoint")
test "up/0 migrates public_endpoint to base_url", %{migration: migration} do
s3_values = [
public_endpoint: "https://coolhost.com/",
bucket: "secret_bucket"
]
insert(:config, group: :pleroma, key: Pleroma.Uploaders.S3, value: s3_values)
upload_values = [
uploader: Pleroma.Uploaders.S3
]
insert(:config, group: :pleroma, key: Pleroma.Upload, value: upload_values)
migration.up()
assert [bucket: "secret_bucket"] ==
ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Uploaders.S3}).value
assert [uploader: Pleroma.Uploaders.S3, base_url: "https://coolhost.com/"] ==
ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}).value
end
test "down/0 reverts base_url to public_endpoint", %{migration: migration} do
s3_values = [
bucket: "secret_bucket"
]
insert(:config, group: :pleroma, key: Pleroma.Uploaders.S3, value: s3_values)
upload_values = [
uploader: Pleroma.Uploaders.S3,
base_url: "https://coolhost.com/"
]
insert(:config, group: :pleroma, key: Pleroma.Upload, value: upload_values)
migration.down()
assert [bucket: "secret_bucket", public_endpoint: "https://coolhost.com/"] ==
ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Uploaders.S3}).value
assert [uploader: Pleroma.Uploaders.S3] ==
ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}).value
end
end

7
test/pleroma/scheduled_activity_test.exs
View file

@ -4,15 +4,14 @@
defmodule Pleroma.ScheduledActivityTest do
use Pleroma.DataCase
alias Pleroma.DataCase
alias Pleroma.ScheduledActivity
import Pleroma.Factory
setup do: clear_config([ScheduledActivity, :enabled])
setup context do
DataCase.ensure_local_uploader(context)
end
setup [:ensure_local_uploader]
describe "creation" do
test "scheduled activities with jobs when ScheduledActivity enabled" do

6
test/pleroma/upload_test.exs
View file

@ -133,7 +133,7 @@ test "returns a media url" do
assert %{"url" => [%{"href" => url}]} = data
assert String.starts_with?(url, Pleroma.Web.base_url() <> "/media/")
assert String.starts_with?(url, Pleroma.Upload.base_url())
end
test "copies the file to the configured folder with deduping" do
@ -148,8 +148,8 @@ test "copies the file to the configured folder with deduping" do
{:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
assert List.first(data["url"])["href"] ==
Pleroma.Web.base_url() <>
"/media/e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781.jpg"
Pleroma.Upload.base_url() <>
"e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781.jpg"
end
test "copies the file to the configured folder without deduping" do

17
test/pleroma/uploaders/s3_test.exs
View file

@ -12,14 +12,10 @@ defmodule Pleroma.Uploaders.S3Test do
import ExUnit.CaptureLog
setup do
clear_config(Pleroma.Upload,
uploader: Pleroma.Uploaders.S3
)
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
clear_config([Pleroma.Upload, :base_url], "https://s3.amazonaws.com")
clear_config([Pleroma.Uploaders.S3])
clear_config([Pleroma.Uploaders.S3, :bucket], "test_bucket")
end
describe "get_file/1" do
@ -33,10 +29,12 @@ test "it returns path to local folder for files" do
test "it returns path without bucket when truncated_namespace set to ''" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
bucket_namespace: "myaccount",
truncated_namespace: ""
)
Config.put([Pleroma.Upload, :base_url], "https://s3.amazonaws.com")
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/test_image.jpg"}
@ -46,7 +44,6 @@ test "it returns path without bucket when truncated_namespace set to ''" do
test "it returns path with bucket namespace when namespace is set" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
bucket_namespace: "family"
)

16
test/pleroma/user/backup_test.exs
View file

@ -195,12 +195,8 @@ test "it creates a zip archive with user data" do
describe "it uploads and deletes a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
clear_config([Pleroma.Upload, :uploader])
clear_config([Pleroma.Upload, :base_url], "https://s3.amazonaws.com")
clear_config([Pleroma.Uploaders.S3, :bucket], "test_bucket")
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
@ -219,7 +215,8 @@ test "it creates a zip archive with user data" do
end
test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
clear_config([Pleroma.Uploaders.S3, :streaming_enabled], false)
with_mock ExAws,
request: fn
@ -229,13 +226,10 @@ test "S3", %{path: path, backup: backup} do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
end
end
test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)

15
test/support/data_case.ex
View file

@ -18,6 +18,8 @@ defmodule Pleroma.DataCase do
use ExUnit.CaseTemplate
import Pleroma.Tests.Helpers, only: [clear_config: 2]
using do
quote do
alias Pleroma.Repo
@ -105,17 +107,10 @@ def stub_pipeline do
end
def ensure_local_uploader(context) do
test_uploader = Map.get(context, :uploader, Pleroma.Uploaders.Local)
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
filters = Pleroma.Config.get([Pleroma.Upload, :filters])
test_uploader = Map.get(context, :uploader) || Pleroma.Uploaders.Local
Pleroma.Config.put([Pleroma.Upload, :uploader], test_uploader)
Pleroma.Config.put([Pleroma.Upload, :filters], [])
on_exit(fn ->
Pleroma.Config.put([Pleroma.Upload, :uploader], uploader)
Pleroma.Config.put([Pleroma.Upload, :filters], filters)
end)
clear_config([Pleroma.Upload, :uploader], test_uploader)
clear_config([Pleroma.Upload, :filters], [])
:ok
end

12
test/support/helpers.ex
View file

@ -8,6 +8,8 @@ defmodule Pleroma.Tests.Helpers do
"""
alias Pleroma.Config
require Logger
defmacro clear_config(config_path) do
quote do
clear_config(unquote(config_path)) do
@ -18,6 +20,7 @@ defmacro clear_config(config_path) do
defmacro clear_config(config_path, do: yield) do
quote do
initial_setting = Config.fetch(unquote(config_path))
unquote(yield)
on_exit(fn ->
@ -35,6 +38,15 @@ defmacro clear_config(config_path, do: yield) do
end
defmacro clear_config(config_path, temp_setting) do
# NOTE: `clear_config([section, key], value)` != `clear_config([section], key: value)` (!)
# Displaying a warning to prevent unintentional clearing of all but one keys in section
if Keyword.keyword?(temp_setting) and length(temp_setting) == 1 do
Logger.warn(
"Please change to `clear_config([section]); clear_config([section, key], value)`: " <>
"#{inspect(config_path)}, #{inspect(temp_setting)}"
)
end
quote do
clear_config(unquote(config_path)) do
Config.put(unquote(config_path), unquote(temp_setting))