Make backup parameters configurable

config/config.exs

@@ -874,7 +874,9 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthent
 config :pleroma, Pleroma.User.Backup,
   purge_after_days: 30,
   limit_days: 7,
-  dir: nil
+  dir: nil,
+  process_wait_time: 30_000,
+  process_chunk_size: 100
 
 config :pleroma, ConcurrentLimiter, [
   {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},

config/description.exs

@@ -3394,6 +3394,21 @@ config :pleroma, :config_description, [
         type: :integer,
         description: "Limit user to export not more often than once per N days",
         suggestions: [7]
+      },
+      %{
+        key: :process_wait_time,
+        type: :integer,
+        label: "Process Wait Time",
+        description:
+          "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed.",
+        suggestions: [30_000]
+      },
+      %{
+        key: :process_chunk_size,
+        type: :integer,
+        label: "Process Chunk Size",
+        description: "The number of activities to fetch in the backup job for each chunk.",
+        suggestions: [100]
       }
     ]
   },

lib/pleroma/user/backup.ex

@@ -35,8 +35,6 @@ defmodule Pleroma.User.Backup do
     timestamps()
   end
 
-  @report_every 100
-
   def create(user, admin_id \\ nil) do
     with :ok <- validate_limit(user, admin_id),
          {:ok, backup} <- user |> new() |> Repo.insert() do
@@ -160,6 +158,8 @@ defmodule Pleroma.User.Backup do
   end
 
   defp wait_backup(backup, current_processed, task) do
+    wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time])
+
     receive do
       {:progress, new_processed} ->
         total_processed = current_processed + new_processed
@@ -175,6 +175,8 @@ defmodule Pleroma.User.Backup do
 
           {:ok, backup} = set_state(backup, :failed)
 
+          cleanup(backup)
+
           {:error,
            %{
              backup: backup,
@@ -185,15 +187,17 @@ defmodule Pleroma.User.Backup do
           {:ok, backup}
         end
     after
-      30_000 ->
+      wait_time ->
         Logger.error(
-          "Backup #{backup.id} timed out after no response for 30 seconds, terminating"
+          "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating"
         )
 
         Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid)
 
         {:ok, backup} = set_state(backup, :failed)
 
+        cleanup(backup)
+
         {:error,
          %{
            backup: backup,
@@ -205,8 +209,7 @@ defmodule Pleroma.User.Backup do
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
   def export(%__MODULE__{} = backup, caller_pid) do
     backup = Repo.preload(backup, :user)
-    name = String.trim_trailing(backup.file_name, ".zip")
-    dir = dir(name)
+    dir = backup_tempdir(backup)
 
     with :ok <- File.mkdir(dir),
          :ok <- actor(dir, backup.user, caller_pid),
@@ -264,16 +267,28 @@ defmodule Pleroma.User.Backup do
     )
   end
 
-  defp should_report?(num), do: rem(num, @report_every) == 0
+  defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0
+
+  defp backup_tempdir(backup) do
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir(name)
+  end
+
+  defp cleanup(backup) do
+    dir = backup_tempdir(backup)
+    File.rm_rf(dir)
+  end
 
   defp write(query, dir, name, fun, caller_pid) do
     path = Path.join(dir, "#{name}.json")
 
+    chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size])
+
     with {:ok, file} <- File.open(path, [:write, :utf8]),
          :ok <- write_header(file, name) do
       total =
         query
-        |> Pleroma.Repo.chunk_stream(100)
+        |> Pleroma.Repo.chunk_stream(chunk_size, _returns_as = :one, timeout: :infinity)
         |> Enum.reduce(0, fn i, acc ->
           with {:ok, data} <-
                  (try do
@@ -283,8 +298,8 @@ defmodule Pleroma.User.Backup do
                   end),
                {:ok, str} <- Jason.encode(data),
                :ok <- IO.write(file, str <> ",\n") do
-            if should_report?(acc + 1) do
-              send(caller_pid, {:progress, @report_every})
+            if should_report?(acc + 1, chunk_size) do
+              send(caller_pid, {:progress, chunk_size})
             end
 
             acc + 1
@@ -301,7 +316,7 @@ defmodule Pleroma.User.Backup do
           end
         end)
 
-      send(caller_pid, {:progress, rem(total, @report_every)})
+      send(caller_pid, {:progress, rem(total, chunk_size)})
 
       with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
         File.close(file)