Merge branch 'feature/uploader_s3' into 'develop'

Uploader.S3 added support stream uploads

See merge request pleroma/pleroma!1545

lib/pleroma/uploaders/s3.ex

@@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger
 
+  alias Pleroma.Config
+
   # The file name is re-encoded with S3's constraints here to comply with previous
   # links with less strict filenames
   def get_file(file) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
@@ -34,15 +36,15 @@ defmodule Pleroma.Uploaders.S3 do
   end
 
   def put_file(%Pleroma.Upload{} = upload) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.get(config, :bucket)
 
-    {:ok, file_data} = File.read(upload.tempfile)
-
     s3_name = strict_encode(upload.path)
 
     op =
-      ExAws.S3.put_object(bucket, s3_name, file_data, [
+      upload.tempfile
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload(bucket, s3_name, [
         {:acl, :public_read},
         {:content_type, upload.content_type}
       ])

mix.exs

@@ -114,8 +114,9 @@ defmodule Pleroma.Mixfile do
       {:tesla, "~> 1.2"},
       {:jason, "~> 1.0"},
       {:mogrify, "~> 0.6.1"},
-      {:ex_aws, "~> 2.0"},
+      {:ex_aws, "~> 2.1"},
       {:ex_aws_s3, "~> 2.0"},
+      {:sweet_xml, "~> 0.6.6"},
       {:earmark, "~> 1.3"},
       {:bbcode, "~> 0.1.1"},
       {:ex_machina, "~> 2.3", only: :test},

mix.lock

@@ -80,6 +80,7 @@
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
+  "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},

test/uploaders/s3_test.exs

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.S3Test do
+  use Pleroma.DataCase
+
+  alias Pleroma.Config
+  alias Pleroma.Uploaders.S3
+
+  import Mock
+  import ExUnit.CaptureLog
+
+  setup do
+    config = Config.get([Pleroma.Uploaders.S3])
+
+    Config.put([Pleroma.Uploaders.S3],
+      bucket: "test_bucket",
+      public_endpoint: "https://s3.amazonaws.com"
+    )
+
+    on_exit(fn ->
+      Config.put([Pleroma.Uploaders.S3], config)
+    end)
+
+    :ok
+  end
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
+             }
+    end
+
+    test "it returns path without bucket when truncated_namespace set to ''" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        truncated_namespace: ""
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_image.jpg"}
+             }
+    end
+
+    test "it returns path with bucket namespace when namespace is set" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        bucket_namespace: "family"
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
+             }
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "image-tet.jpg",
+        content_type: "image/jpg",
+        path: "test_folder/image-tet.jpg",
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+        assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
+      end
+    end
+
+    test "returns error", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
+        assert capture_log(fn ->
+                 assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
+      end
+    end
+  end
+end