Merge branch 'features/ingestion-page' into 'develop'

Pipeline Ingestion: Page

See merge request pleroma/pleroma!3097

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -91,7 +91,7 @@ defp increase_replies_count_if_reply(%{
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note Page]
   @impl true
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do

lib/pleroma/web/activity_pub/object_validator.ex

@@ -20,7 +20,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
@@ -102,7 +102,7 @@ def validate(
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Video Event Article Note] do
+      when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -115,15 +115,16 @@ def validate(
   end
 
   def validate(%{"type" => type} = object, meta)
-      when type in ~w[Event Question Audio Video Article Note] do
+      when type in ~w[Event Question Audio Video Article Note Page] do
     validator =
       case type do
         "Event" -> EventValidator
         "Question" -> QuestionValidator
         "Audio" -> AudioVideoValidator
         "Video" -> AudioVideoValidator
-        "Article" -> ArticleNoteValidator
-        "Note" -> ArticleNoteValidator
+        "Article" -> ArticleNotePageValidator
+        "Note" -> ArticleNotePageValidator
+        "Page" -> ArticleNotePageValidator
       end
 
     with {:ok, object} <-
@@ -197,8 +198,8 @@ def cast_and_apply(%{"type" => "Event"} = object) do
     EventValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do
-    ArticleNoteValidator.cast_and_apply(object)
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note Page] do
+    ArticleNotePageValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
@@ -113,7 +113,7 @@ def changeset(struct, data) do
 
   defp validate_data(data_cng) do
     data_cng
-    |> validate_inclusion(:type, ["Article", "Note"])
+    |> validate_inclusion(:type, ["Article", "Note", "Page"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])

lib/pleroma/web/activity_pub/side_effects.ex

@@ -437,7 +437,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Video Question Event Article Note] do
+      when objtype in ~w[Audio Video Question Event Article Note Page] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -353,29 +353,6 @@ defp get_reported(objects) do
     end)
   end
 
-  # Compatibility wrapper for Mastodon votes
-  defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do
-    handle_incoming(data)
-  end
-
-  defp handle_create(%{"object" => object} = data, user) do
-    %{
-      to: data["to"],
-      object: object,
-      actor: user,
-      context: object["context"],
-      local: false,
-      published: data["published"],
-      additional:
-        Map.take(data, [
-          "cc",
-          "directMessage",
-          "id"
-        ])
-    }
-    |> ActivityPub.create()
-  end
-
   def handle_incoming(data, options \\ [])
 
   # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
@@ -407,43 +384,6 @@ def handle_incoming(%{"id" => ""}, _options), do: :error
   def handle_incoming(%{"id" => id}, _options) when is_binary(id) and byte_size(id) < 8,
     do: :error
 
-  # TODO: validate those with a Ecto scheme
-  # - tags
-  # - emoji
-  def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data,
-        options
-      ) do
-    actor = Containment.get_actor(data)
-
-    with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do
-      data =
-        data
-        |> Map.put("object", fix_object(object, options))
-        |> Map.put("actor", actor)
-        |> fix_addressing()
-
-      with {:ok, created_activity} <- handle_create(data, user) do
-        reply_depth = (options[:depth] || 0) + 1
-
-        if Federator.allowed_thread_distance?(reply_depth) do
-          for reply_id <- replies(object) do
-            Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
-              "id" => reply_id,
-              "depth" => reply_depth
-            })
-          end
-        end
-
-        {:ok, created_activity}
-      end
-    else
-      %Activity{} = activity -> {:ok, activity}
-      _e -> :error
-    end
-  end
-
   def handle_incoming(
         %{"type" => "Listen", "object" => %{"type" => "Audio"} = object} = data,
         options
@@ -507,7 +447,7 @@ def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
         options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note Page} do
     fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
 
     object =

test/fixtures/tesla_mock/lemmy-page.json

@@ -0,0 +1,17 @@
+{
+  "commentsEnabled": true,
+  "sensitive": false,
+  "stickied": false,
+  "attributedTo": "https://enterprise.lemmy.ml/u/nutomic",
+  "summary": "Hello Federation!",
+  "url": "https://enterprise.lemmy.ml/pictrs/image/US52d9DPvf.jpg",
+  "image": {
+    "type": "Image",
+    "url": "https://enterprise.lemmy.ml/pictrs/image/lwFAcXHUjS.jpg"
+  },
+  "published": "2020-09-14T15:03:11.909105+00:00",
+  "to": "https://enterprise.lemmy.ml/c/main",
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "id": "https://enterprise.lemmy.ml/post/3",
+  "type": "Page"
+}

test/fixtures/tesla_mock/lemmy-user.json

@@ -0,0 +1,27 @@
+{
+  "publicKey": {
+    "id": "https://enterprise.lemmy.ml/u/nutomic#main-key",
+    "owner": "https://enterprise.lemmy.ml/u/nutomic",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvfwAYPxp1gOk2HcCRoUd\nupoecvmnpzRc5Gu6/N3YQyOyRsrYuiYLNQq2cgM3kcU80ZeEetkwkYgXkRJOKu/b\nBWb7i1zt2tdr5k6lUdW8dfCyjht8ooFPQdov8J3QYHfgBHyUYxuCNfSujryxx2wu\nLQcdjRQa5NIWcomSO8OXmCF5/Yhg2XWCbtnlxEq6Y+AFddr1mAlTOy5pBr5d+xZz\njLw/U3CioNJ79yGi/sJhgp6IyJqtUSoN3b4BgRIEts2QVvn44W1rQy9wCbRYQrO1\nBcB9Wel4k3rJJK8uHg+LpHVMaZppkNaWGkMBhMbzr8qmIlcNWNi7cbMK/p5vyviy\nSwIDAQAB\n-----END PUBLIC KEY-----\n"
+  },
+  "inbox": "https://enterprise.lemmy.ml/u/nutomic/inbox",
+  "preferredUsername": "Nutomic",
+  "endpoints": {
+    "sharedInbox": "https://enterprise.lemmy.ml/inbox"
+  },
+  "summary": "some bio",
+  "icon": {
+    "type": "Image",
+    "url": "https://enterprise.lemmy.ml/pictrs/image/F6Z7QcWZRJ.jpg"
+  },
+  "image": {
+    "type": "Image",
+    "url": "https://enterprise.lemmy.ml:/pictrs/image/Q79N9oCDEG.png"
+  },
+  "published": "2020-09-14T14:54:53.080949+00:00",
+  "updated": "2020-10-14T10:58:28.139178+00:00",
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "id": "https://enterprise.lemmy.ml/u/nutomic",
+  "type": "Person",
+  "name": "nutomic"
+}

test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs

@@ -2,10 +2,10 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do
   use Pleroma.DataCase, async: true
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator
   alias Pleroma.Web.ActivityPub.Utils
 
   import Pleroma.Factory
@@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do
     end
 
     test "a basic note validates", %{note: note} do
-      %{valid?: true} = ArticleNoteValidator.cast_and_validate(note)
+      %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
     end
   end
 end

test/pleroma/web/activity_pub/transmogrifier/page_handling_test.exs

@@ -0,0 +1,36 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.PageHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  alias Pleroma.Object.Fetcher
+
+  test "Lemmy Page" do
+    Tesla.Mock.mock(fn
+      %{url: "https://enterprise.lemmy.ml/post/3"} ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-type", "application/activity+json"}],
+          body: File.read!("test/fixtures/tesla_mock/lemmy-page.json")
+        }
+
+      %{url: "https://enterprise.lemmy.ml/u/nutomic"} ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-type", "application/activity+json"}],
+          body: File.read!("test/fixtures/tesla_mock/lemmy-user.json")
+        }
+    end)
+
+    {:ok, object} = Fetcher.fetch_object_from_id("https://enterprise.lemmy.ml/post/3")
+
+    assert object.data["summary"] == "Hello Federation!"
+    assert object.data["published"] == "2020-09-14T15:03:11.909105Z"
+
+    # WAT
+    assert object.data["url"] == "https://enterprise.lemmy.ml/pictrs/image/US52d9DPvf.jpg"
+  end
+end