Merge branch 'proxy-range-and-chunk' into 'develop'

ReverseProxy: Streaming and disable encoding if Range

Closes #1860 and #1823

See merge request pleroma/pleroma!2749

lib/pleroma/reverse_proxy/reverse_proxy.ex

@@ -3,12 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
+  @range_headers ~w(range if-range)
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
-                      ~w(if-unmodified-since if-none-match if-range range)
+                      ~w(if-unmodified-since if-none-match) ++ @range_headers
   @resp_cache_headers ~w(etag date last-modified)
   @keep_resp_headers @resp_cache_headers ++
-                       ~w(content-type content-disposition content-encoding content-range) ++
+                       ~w(content-length content-type content-disposition content-encoding) ++
-                       ~w(accept-ranges vary)
+                       ~w(content-range accept-ranges vary)
   @default_cache_control_header "public, max-age=1209600"
   @valid_resp_codes [200, 206, 304]
   @max_read_duration :timer.seconds(30)
@@ -170,6 +171,8 @@ defmodule Pleroma.ReverseProxy do
   end
 
   defp response(conn, client, url, status, headers, opts) do
+    Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
+
     result =
       conn
       |> put_resp_headers(build_resp_headers(headers, opts))
@@ -220,7 +223,9 @@ defmodule Pleroma.ReverseProxy do
     end
   end
 
-  defp head_response(conn, _url, code, headers, opts) do
+  defp head_response(conn, url, code, headers, opts) do
+    Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
+
     conn
     |> put_resp_headers(build_resp_headers(headers, opts))
     |> send_resp(code, "")
@@ -262,20 +267,33 @@ defmodule Pleroma.ReverseProxy do
     headers
     |> downcase_headers()
     |> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
-    |> (fn headers ->
+    |> build_req_range_or_encoding_header(opts)
-          headers = headers ++ Keyword.get(opts, :req_headers, [])
+    |> build_req_user_agent_header(opts)
+    |> Keyword.merge(Keyword.get(opts, :req_headers, []))
+  end
 
-          if Keyword.get(opts, :keep_user_agent, false) do
+  # Disable content-encoding if any @range_headers are requested (see #1823).
-            List.keystore(
+  defp build_req_range_or_encoding_header(headers, _opts) do
-              headers,
+    range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
-              "user-agent",
+
-              0,
+    if range? && List.keymember?(headers, "accept-encoding", 0) do
-              {"user-agent", Pleroma.Application.user_agent()}
+      List.keydelete(headers, "accept-encoding", 0)
-            )
+    else
-          else
+      headers
-            headers
+    end
-          end
+  end
-        end).()
+
+  defp build_req_user_agent_header(headers, opts) do
+    if Keyword.get(opts, :keep_user_agent, false) do
+      List.keystore(
+        headers,
+        "user-agent",
+        0,
+        {"user-agent", Pleroma.Application.user_agent()}
+      )
+    else
+      headers
+    end
   end
 
   defp build_resp_headers(headers, opts) do
@@ -283,7 +301,7 @@ defmodule Pleroma.ReverseProxy do
     |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
     |> build_resp_cache_headers(opts)
     |> build_resp_content_disposition_header(opts)
-    |> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).()
+    |> Keyword.merge(Keyword.get(opts, :resp_headers, []))
   end
 
   defp build_resp_cache_headers(headers, _opts) do

test/reverse_proxy/reverse_proxy_test.exs

@@ -314,7 +314,7 @@ defmodule Pleroma.ReverseProxyTest do
     test "not atachment", %{conn: conn} do
       disposition_headers_mock([
         {"content-type", "image/gif"},
-        {"content-length", 0}
+        {"content-length", "0"}
       ])
 
       conn = ReverseProxy.call(conn, "/disposition")
@@ -325,7 +325,7 @@ defmodule Pleroma.ReverseProxyTest do
     test "with content-disposition header", %{conn: conn} do
       disposition_headers_mock([
         {"content-disposition", "attachment; filename=\"filename.jpg\""},
-        {"content-length", 0}
+        {"content-length", "0"}
       ])
 
       conn = ReverseProxy.call(conn, "/disposition")