Merge branch 'issue/1798' into 'develop'

[#1798] fixed migration the settings to DB

See merge request pleroma/pleroma!2655

lib/mix/tasks/pleroma/config.ex

@@ -52,6 +52,7 @@ def migrate_to_db(file_path \\ nil) do
 
   defp do_migrate_to_db(config_file) do
     if File.exists?(config_file) do
+      shell_info("Migrating settings from file: #{Path.expand(config_file)}")
       Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
       Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
 

lib/pleroma/application.ex

@@ -39,7 +39,7 @@ def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
     Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
-    Pleroma.Repo.check_migrations_applied!()
+    Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
 

lib/pleroma/application_requirements.ex

@@ -0,0 +1,107 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ApplicationRequirements do
+  @moduledoc """
+  The module represents the collection of validations to runs before start server.
+  """
+
+  defmodule VerifyError, do: defexception([:message])
+
+  import Ecto.Query
+
+  require Logger
+
+  @spec verify!() :: :ok | VerifyError.t()
+  def verify! do
+    :ok
+    |> check_migrations_applied!()
+    |> check_rum!()
+    |> handle_result()
+  end
+
+  defp handle_result(:ok), do: :ok
+  defp handle_result({:error, message}), do: raise(VerifyError, message: message)
+
+  # Checks for pending migrations.
+  #
+  def check_migrations_applied!(:ok) do
+    unless Pleroma.Config.get(
+             [:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
+             false
+           ) do
+      {_, res, _} =
+        Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+          down_migrations =
+            Ecto.Migrator.migrations(repo)
+            |> Enum.reject(fn
+              {:up, _, _} -> true
+              {:down, _, _} -> false
+            end)
+
+          if length(down_migrations) > 0 do
+            down_migrations_text =
+              Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
+
+            Logger.error(
+              "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
+            )
+
+            {:error, "Unapplied Migrations detected"}
+          else
+            :ok
+          end
+        end)
+
+      res
+    else
+      :ok
+    end
+  end
+
+  def check_migrations_applied!(result), do: result
+
+  # Checks for settings of RUM indexes.
+  #
+  defp check_rum!(:ok) do
+    {_, res, _} =
+      Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+        migrate =
+          from(o in "columns",
+            where: o.table_name == "objects",
+            where: o.column_name == "fts_content"
+          )
+          |> repo.exists?(prefix: "information_schema")
+
+        setting = Pleroma.Config.get([:database, :rum_enabled], false)
+
+        do_check_rum!(setting, migrate)
+      end)
+
+    res
+  end
+
+  defp check_rum!(result), do: result
+
+  defp do_check_rum!(setting, migrate) do
+    case {setting, migrate} do
+      {true, false} ->
+        Logger.error(
+          "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+        )
+
+        {:error, "Unapplied RUM Migrations detected"}
+
+      {false, true} ->
+        Logger.error(
+          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+        )
+
+        {:error, "RUM Migrations detected"}
+
+      _ ->
+        :ok
+    end
+  end
+end

lib/pleroma/repo.ex

@@ -11,9 +11,7 @@ defmodule Pleroma.Repo do
   import Ecto.Query
   require Logger
 
-  defmodule Instrumenter do
-    use Prometheus.EctoInstrumenter
-  end
+  defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
 
   @doc """
   Dynamically loads the repository url from the
@@ -51,35 +49,6 @@ def get_assoc(resource, association) do
     end
   end
 
-  def check_migrations_applied!() do
-    unless Pleroma.Config.get(
-             [:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
-             false
-           ) do
-      Ecto.Migrator.with_repo(__MODULE__, fn repo ->
-        down_migrations =
-          Ecto.Migrator.migrations(repo)
-          |> Enum.reject(fn
-            {:up, _, _} -> true
-            {:down, _, _} -> false
-          end)
-
-        if length(down_migrations) > 0 do
-          down_migrations_text =
-            Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
-
-          Logger.error(
-            "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
-          )
-
-          raise Pleroma.Repo.UnappliedMigrationsError
-        end
-      end)
-    else
-      :ok
-    end
-  end
-
   def chunk_stream(query, chunk_size) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
@@ -107,7 +76,3 @@ def chunk_stream(query, chunk_size) do
     )
   end
 end
-
-defmodule Pleroma.Repo.UnappliedMigrationsError do
-  defexception message: "Unapplied Migrations detected"
-end

priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs

@@ -10,8 +10,8 @@ def up do
 
     execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
     begin
-      new.fts_content := to_tsvector('english', new.data->>'content');
-      return new;
+    new.fts_content := to_tsvector('english', new.data->>'content');
+    return new;
     end
     $$ LANGUAGE plpgsql")
     execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")

test/application_requirements_test.exs

@@ -0,0 +1,96 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ApplicationRequirementsTest do
+  use Pleroma.DataCase
+  import ExUnit.CaptureLog
+  import Mock
+
+  alias Pleroma.Repo
+
+  describe "check_rum!" do
+    setup_with_mocks([
+      {Pleroma.ApplicationRequirements, [:passthrough],
+       [check_migrations_applied!: fn _ -> :ok end]}
+    ]) do
+      :ok
+    end
+
+    setup do: clear_config([:database, :rum_enabled])
+
+    test "raises if rum is enabled and detects unapplied rum migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], true)
+
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
+        assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                     "Unapplied RUM Migrations detected",
+                     fn ->
+                       capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     end
+      end
+    end
+
+    test "raises if rum is disabled and detects rum migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], false)
+
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
+        assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                     "RUM Migrations detected",
+                     fn ->
+                       capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     end
+      end
+    end
+
+    test "doesn't do anything if rum enabled and applied migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], true)
+
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
+        assert Pleroma.ApplicationRequirements.verify!() == :ok
+      end
+    end
+
+    test "doesn't do anything if rum disabled" do
+      Pleroma.Config.put([:database, :rum_enabled], false)
+
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
+        assert Pleroma.ApplicationRequirements.verify!() == :ok
+      end
+    end
+  end
+
+  describe "check_migrations_applied!" do
+    setup_with_mocks([
+      {Ecto.Migrator, [],
+       [
+         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
+         migrations: fn Repo ->
+           [
+             {:up, 20_191_128_153_944, "fix_missing_following_count"},
+             {:up, 20_191_203_043_610, "create_report_notes"},
+             {:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
+           ]
+         end
+       ]}
+    ]) do
+      :ok
+    end
+
+    setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
+
+    test "raises if it detects unapplied migrations" do
+      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                   "Unapplied Migrations detected",
+                   fn ->
+                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                   end
+    end
+
+    test "doesn't do anything if disabled" do
+      Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
+
+      assert :ok == Pleroma.ApplicationRequirements.verify!()
+    end
+  end
+end

test/fixtures/config/temp.secret.exs

@@ -9,3 +9,5 @@
 config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox
 
 config :postgrex, :json_library, Poison
+
+config :pleroma, :database, rum_enabled: true

test/repo_test.exs

@@ -4,9 +4,7 @@
 
 defmodule Pleroma.RepoTest do
   use Pleroma.DataCase
-  import ExUnit.CaptureLog
   import Pleroma.Factory
-  import Mock
 
   alias Pleroma.User
 
@@ -49,36 +47,4 @@ test "return error if has not assoc " do
       assert Repo.get_assoc(token, :user) == {:error, :not_found}
     end
   end
-
-  describe "check_migrations_applied!" do
-    setup_with_mocks([
-      {Ecto.Migrator, [],
-       [
-         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
-         migrations: fn Pleroma.Repo ->
-           [
-             {:up, 20_191_128_153_944, "fix_missing_following_count"},
-             {:up, 20_191_203_043_610, "create_report_notes"},
-             {:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
-           ]
-         end
-       ]}
-    ]) do
-      :ok
-    end
-
-    setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
-
-    test "raises if it detects unapplied migrations" do
-      assert_raise Pleroma.Repo.UnappliedMigrationsError, fn ->
-        capture_log(&Repo.check_migrations_applied!/0)
-      end
-    end
-
-    test "doesn't do anything if disabled" do
-      Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
-
-      assert :ok == Repo.check_migrations_applied!()
-    end
-  end
 end

test/tasks/config_test.exs

@@ -50,6 +50,7 @@ test "filtered settings are migrated to db" do
       config3 = ConfigDB.get_by_params(%{group: ":quack", key: ":level"})
       refute ConfigDB.get_by_params(%{group: ":pleroma", key: "Pleroma.Repo"})
       refute ConfigDB.get_by_params(%{group: ":postgrex", key: ":json_library"})
+      refute ConfigDB.get_by_params(%{group: ":pleroma", key: ":database"})
 
       assert config1.value == [key: "value", key2: [Repo]]
       assert config2.value == [key: "value2", key2: ["Activity"]]