[feature] Media cleanup endpoint (#560)

Adds an admin endpoint to trigger a remote media cleanup.

Fixed #348

Signed-off-by: Sashanoraa <sasha@noraa.gay>

docs/api/swagger.yaml

@@ -2574,6 +2574,38 @@ paths:
       summary: View domain block with the given ID.
       tags:
       - admin
+  /api/v1/admin/media_cleanup:
+    post:
+      consumes:
+      - application/json
+      - application/xml
+      - application/x-www-form-urlencoded
+      operationId: mediaCleanup
+      parameters:
+      - description: |-
+          Number of days of remote media to keep. Native values will be treated as 0.
+          If value is not specified, the value of media-remote-cache-days in the server config will be used.
+        format: int64
+        in: query
+        name: remote_cache_days
+        type: integer
+        x-go-name: RemoteCacheDays
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: Echos the number of days requested. The cleanup is performed
+            asynchronously after the request completes.
+        "400":
+          description: bad request
+        "403":
+          description: forbidden
+      security:
+      - OAuth2 Bearer:
+        - admin
+      summary: Clean up remote media older than the specified number of days.
+      tags:
+      - admin
   /api/v1/apps:
     post:
       consumes:

internal/api/client/admin/admin.go

@@ -41,6 +41,7 @@ const (
 	AccountsPathWithID = AccountsPath + "/:" + IDKey
 	// AccountsActionPath is used for taking action on a single account.
 	AccountsActionPath = AccountsPathWithID + "/action"
+	MediaCleanupPath   = BasePath + "/media_cleanup"
 
 	// ExportQueryKey is for requesting a public export of some data.
 	ExportQueryKey = "export"
@@ -70,5 +71,6 @@ func (m *Module) Route(r router.Router) error {
 	r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler)
 	r.AttachHandler(http.MethodDelete, DomainBlocksPathWithID, m.DomainBlockDELETEHandler)
 	r.AttachHandler(http.MethodPost, AccountsActionPath, m.AccountActionPOSTHandler)
+	r.AttachHandler(http.MethodPost, MediaCleanupPath, m.MediaCleanupPOSTHandler)
 	return nil
 }

internal/api/client/admin/mediacleanup.go

@@ -0,0 +1,110 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// MediaCleanupPOSTHandler swagger:operation POST /api/v1/admin/media_cleanup mediaCleanup
+//
+// Clean up remote media older than the specified number of days.
+//
+// ---
+// tags:
+// - admin
+//
+// consumes:
+// - application/json
+// - application/xml
+// - application/x-www-form-urlencoded
+//
+// produces:
+// - application/json
+//
+// security:
+// - OAuth2 Bearer:
+//   - admin
+//
+// responses:
+//   '200':
+//     description: |-
+//      Echos the number of days requested. The cleanup is performed asynchronously after the request completes.
+//   '403':
+//      description: forbidden
+//   '400':
+//      description: bad request
+func (m *Module) MediaCleanupPOSTHandler(c *gin.Context) {
+	l := logrus.WithFields(logrus.Fields{
+		"func":        "MediaCleanupPOSTHandler",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+
+	// make sure we're authed...
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		l.Debugf("couldn't auth: %s", err)
+		c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+		return
+	}
+
+	// with an admin account
+	if !authed.User.Admin {
+		l.Debugf("user %s not an admin", authed.User.ID)
+		c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"})
+		return
+	}
+
+	// extract the form from the request context
+	l.Tracef("parsing request form: %+v", c.Request.Form)
+	form := &model.MediaCleanupRequest{}
+	if err := c.ShouldBind(form); err != nil {
+		l.Debugf("error parsing form %+v: %s", c.Request.Form, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)})
+		return
+	}
+
+	var remoteCacheDays int
+	if form.RemoteCacheDays == nil {
+		remoteCacheDays = viper.GetInt(config.Keys.MediaRemoteCacheDays)
+	} else {
+		remoteCacheDays = *form.RemoteCacheDays
+	}
+	if remoteCacheDays < 0 {
+		remoteCacheDays = 0
+	}
+
+	if errWithCode := m.processor.AdminMediaRemotePrune(c.Request.Context(), remoteCacheDays); errWithCode != nil {
+		l.Debugf("error starting prune of remote media: %s", errWithCode.Error())
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+		return
+	}
+
+	c.JSON(http.StatusOK, remoteCacheDays)
+}

internal/api/client/admin/mediacleanup_test.go

@@ -0,0 +1,114 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+)
+
+type MediaCleanupTestSuite struct {
+	AdminStandardTestSuite
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanup() {
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+	suite.True(testAttachment.Cached)
+
+	// set up the request
+	recorder := httptest.NewRecorder()
+	ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 1}"), admin.EmojiPath, "application/json")
+
+	// call the handler
+	suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+	// we should have OK because our request was valid
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Wait for async task to finish
+	time.Sleep(1 * time.Second)
+
+	// Get media we prunes
+	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	suite.NoError(err)
+
+	// the media should no longer be cached
+	suite.False(prunedAttachment.Cached)
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanupNoArg() {
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+	suite.True(testAttachment.Cached)
+	println("TIME: ", testAttachment.CreatedAt.String())
+
+	// set up the request
+	recorder := httptest.NewRecorder()
+	ctx := suite.newContext(recorder, http.MethodPost, []byte("{}"), admin.EmojiPath, "application/json")
+
+	// call the handler
+	suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+	// we should have OK because our request was valid
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Wait for async task to finish
+	time.Sleep(1 * time.Second)
+
+	// Get media we prunes
+	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	suite.NoError(err)
+
+	// the media should no longer be cached
+	suite.True(prunedAttachment.Cached)
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanupNotOldEnough() {
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+	suite.True(testAttachment.Cached)
+
+	// set up the request
+	recorder := httptest.NewRecorder()
+	ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 3}"), admin.EmojiPath, "application/json")
+
+	// call the handler
+	suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+	// we should have OK because our request was valid
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Wait for async task to finish
+	time.Sleep(1 * time.Second)
+
+	// Get media we prunes
+	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	suite.NoError(err)
+
+	// the media should still be cached
+	suite.True(prunedAttachment.Cached)
+}
+
+func TestMediaCleanupTestSuite(t *testing.T) {
+	suite.Run(t, &MediaCleanupTestSuite{})
+}

internal/api/model/admin.go

@@ -91,3 +91,12 @@ type AdminAccountActionRequest struct {
 	// ID of the account to be acted on.
 	TargetAccountID string `form:"-" json:"-" xml:"-"`
 }
+
+// MediaCleanupRequest models admin media cleanup parameters
+//
+// swagger:parameters mediaCleanup
+type MediaCleanupRequest struct {
+	// Number of days of remote media to keep. Native values will be treated as 0.
+	// If value is not specified, the value of media-remote-cache-days in the server config will be used.
+	RemoteCacheDays *int `form:"remote_cache_days" json:"remote_cache_days" xml:"remote_cache_days"`
+}

internal/db/bundb/media_test.go

@@ -40,7 +40,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() {
 func (suite *MediaTestSuite) TestGetOlder() {
 	attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
 	suite.NoError(err)
-	suite.Len(attachments, 1)
+	suite.Len(attachments, 2)
 }
 
 func TestMediaTestSuite(t *testing.T) {

internal/media/pruneremote_test.go

@@ -39,7 +39,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
 
 	totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
 	suite.NoError(err)
-	suite.Equal(1, totalPruned)
+	suite.Equal(2, totalPruned)
 
 	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
 	suite.NoError(err)
@@ -51,7 +51,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
 func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
 	totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
 	suite.NoError(err)
-	suite.Equal(1, totalPruned)
+	suite.Equal(2, totalPruned)
 
 	// final prune should prune nothing, since the first prune already happened
 	totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
@@ -65,7 +65,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
 
 	totalPruned, err := suite.manager.PruneRemote(ctx, 1)
 	suite.NoError(err)
-	suite.Equal(1, totalPruned)
+	suite.Equal(2, totalPruned)
 
 	// media should no longer be stored
 	_, err = suite.storage.Get(testAttachment.File.Path)
@@ -118,7 +118,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
 	// Now attempt to prune remote for item with db entry no file
 	totalPruned, err := suite.manager.PruneRemote(ctx, 1)
 	suite.NoError(err)
-	suite.Equal(1, totalPruned)
+	suite.Equal(2, totalPruned)
 }
 
 func TestPruneRemoteTestSuite(t *testing.T) {

testrig/testmodels.go

@@ -831,6 +831,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Header: false,
 			Cached: true,
 		},
+		"remote_account_1_status_1_attachment_2": {
+			ID:        "01FVW7RXPQ8YJHTEXYPE7Q8ZY1",
+			StatusID:  "01FVW7JHQFSFK166WWKR8CBA6M",
+			URL:       "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+			RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
+			CreatedAt: time.Now().Add(-48 * time.Hour),
+			UpdatedAt: time.Now().Add(-48 * time.Hour),
+			Type:      gtsmodel.FileTypeImage,
+			FileMeta: gtsmodel.FileMeta{
+				Original: gtsmodel.Original{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Small: gtsmodel.Small{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Focus: gtsmodel.Focus{
+					X: 0,
+					Y: 0,
+				},
+			},
+			AccountID:         "01F8MH1H7YV1Z7D2C8K2730QBF",
+			Description:       "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
+			ScheduledStatusID: "",
+			Blurhash:          "LARysgM_IU_3~pD%M_Rj_39FIAt6",
+			Processing:        2,
+			File: gtsmodel.File{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    19310,
+				UpdatedAt:   time.Now().Add(-48 * time.Hour),
+			},
+			Thumbnail: gtsmodel.Thumbnail{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    20395,
+				UpdatedAt:   time.Now().Add(-48 * time.Hour),
+				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				RemoteURL:   "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
+			},
+			Avatar: false,
+			Header: false,
+			Cached: true,
+		},
 	}
 }