gotosocial/internal/media/prune_test.go
kim 5faeb4de20
[chore] tidy up media manager, add calling func to errors, build-script improvements (#1835)
* media manager tidy-up: de-interface and remove unused PostDataFunc

Signed-off-by: kim <grufwub@gmail.com>

* remove last traces of media.Manager being an interface

Signed-off-by: kim <grufwub@gmail.com>

* update error to provide caller, allow tuneable via build tags

Signed-off-by: kim <grufwub@gmail.com>

* remove kim-specific build script changes

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts

Signed-off-by: kim <grufwub@gmail.com>

* update build-script to support externally setting build variables

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>
2023-05-28 14:08:35 +02:00

358 lines
12 KiB
Go

// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package media_test
import (
"bytes"
"context"
"io"
"os"
"testing"
"codeberg.org/gruf/go-store/v2/storage"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type PruneTestSuite struct {
MediaStandardTestSuite
}
func (suite *PruneTestSuite) TestPruneOrphanedDry() {
// add a big orphan panda to store
b, err := os.ReadFile("./test/big-panda.gif")
if err != nil {
suite.FailNow(err.Error())
}
pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil {
suite.FailNow(err.Error())
}
// dry run should show up 1 orphaned panda
totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true)
suite.NoError(err)
suite.Equal(1, totalPruned)
// panda should still be in storage
hasKey, err := suite.storage.Has(context.Background(), pandaPath)
suite.NoError(err)
suite.True(hasKey)
}
func (suite *PruneTestSuite) TestPruneOrphanedMoist() {
// add a big orphan panda to store
b, err := os.ReadFile("./test/big-panda.gif")
if err != nil {
suite.FailNow(err.Error())
}
pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil {
suite.FailNow(err.Error())
}
// should show up 1 orphaned panda
totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false)
suite.NoError(err)
suite.Equal(1, totalPruned)
// panda should no longer be in storage
hasKey, err := suite.storage.Has(context.Background(), pandaPath)
suite.NoError(err)
suite.False(hasKey)
}
func (suite *PruneTestSuite) TestPruneUnusedLocal() {
testAttachment := suite.testAttachments["local_account_1_unattached_1"]
suite.True(*testAttachment.Cached)
totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false)
suite.NoError(err)
suite.Equal(1, totalPruned)
_, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.ErrorIs(err, db.ErrNoEntries)
}
func (suite *PruneTestSuite) TestPruneUnusedLocalDry() {
testAttachment := suite.testAttachments["local_account_1_unattached_1"]
suite.True(*testAttachment.Cached)
totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true)
suite.NoError(err)
suite.Equal(1, totalPruned)
_, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.NoError(err)
}
func (suite *PruneTestSuite) TestPruneRemoteTwice() {
totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false)
suite.NoError(err)
suite.Equal(1, totalPruned)
// final prune should prune nothing, since the first prune already happened
totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false)
suite.NoError(err)
suite.Equal(0, totalPrunedAgain)
}
func (suite *PruneTestSuite) TestPruneOneNonExistent() {
ctx := context.Background()
testAttachment := suite.testAttachments["local_account_1_unattached_1"]
// Delete this attachment cached on disk
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(*media.Cached)
err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)
// Now attempt to prune for item with db entry no file
totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false)
suite.NoError(err)
suite.Equal(1, totalPruned)
}
func (suite *PruneTestSuite) TestPruneUnusedRemote() {
ctx := context.Background()
// start by clearing zork's avatar + header
zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
zork := suite.testAccounts["local_account_1"]
zork.AvatarMediaAttachmentID = ""
zork.HeaderMediaAttachmentID = ""
if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
panic(err)
}
totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
suite.NoError(err)
suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
_, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
suite.ErrorIs(err, db.ErrNoEntries)
_, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
suite.ErrorIs(err, db.ErrNoEntries)
}
func (suite *PruneTestSuite) TestPruneUnusedRemoteTwice() {
ctx := context.Background()
// start by clearing zork's avatar + header
zork := suite.testAccounts["local_account_1"]
zork.AvatarMediaAttachmentID = ""
zork.HeaderMediaAttachmentID = ""
if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
panic(err)
}
totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
suite.NoError(err)
suite.Equal(2, totalPruned)
// final prune should prune nothing, since the first prune already happened
totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false)
suite.NoError(err)
suite.Equal(0, totalPruned)
}
func (suite *PruneTestSuite) TestPruneUnusedRemoteMultipleAccounts() {
ctx := context.Background()
// start by clearing zork's avatar + header
zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
zork := suite.testAccounts["local_account_1"]
zork.AvatarMediaAttachmentID = ""
zork.HeaderMediaAttachmentID = ""
if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
panic(err)
}
// set zork's unused header as belonging to turtle
turtle := suite.testAccounts["local_account_1"]
zorkOldHeader.AccountID = turtle.ID
if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil {
panic(err)
}
totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
suite.NoError(err)
suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
_, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
suite.ErrorIs(err, db.ErrNoEntries)
_, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
suite.ErrorIs(err, db.ErrNoEntries)
}
func (suite *PruneTestSuite) TestUncacheRemote() {
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
suite.True(*testStatusAttachment.Cached)
testHeader := suite.testAttachments["remote_account_3_header"]
suite.True(*testHeader.Cached)
totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false)
suite.NoError(err)
suite.Equal(2, totalUncached)
uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID)
suite.NoError(err)
suite.False(*uncachedAttachment.Cached)
uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID)
suite.NoError(err)
suite.False(*uncachedAttachment.Cached)
}
func (suite *PruneTestSuite) TestUncacheRemoteDry() {
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
suite.True(*testStatusAttachment.Cached)
testHeader := suite.testAttachments["remote_account_3_header"]
suite.True(*testHeader.Cached)
totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, true)
suite.NoError(err)
suite.Equal(2, totalUncached)
uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID)
suite.NoError(err)
suite.True(*uncachedAttachment.Cached)
uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID)
suite.NoError(err)
suite.True(*uncachedAttachment.Cached)
}
func (suite *PruneTestSuite) TestUncacheRemoteTwice() {
totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false)
suite.NoError(err)
suite.Equal(2, totalUncached)
// final uncache should uncache nothing, since the first uncache already happened
totalUncachedAgain, err := suite.manager.UncacheRemote(context.Background(), 1, false)
suite.NoError(err)
suite.Equal(0, totalUncachedAgain)
}
func (suite *PruneTestSuite) TestUncacheAndRecache() {
ctx := context.Background()
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
testHeader := suite.testAttachments["remote_account_3_header"]
totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false)
suite.NoError(err)
suite.Equal(2, totalUncached)
// media should no longer be stored
_, err = suite.storage.Get(ctx, testStatusAttachment.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, testHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
data := func(_ context.Context) (io.ReadCloser, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg")
if err != nil {
panic(err)
}
return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
}
for _, original := range []*gtsmodel.MediaAttachment{
testStatusAttachment,
testHeader,
} {
processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, original.ID)
suite.NoError(err)
// synchronously load the recached attachment
recachedAttachment, err := processingRecache.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(recachedAttachment)
// recachedAttachment should be basically the same as the old attachment
suite.True(*recachedAttachment.Cached)
suite.Equal(original.ID, recachedAttachment.ID)
suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place
suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
// recached files should be back in storage
_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
suite.NoError(err)
_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}
}
func (suite *PruneTestSuite) TestUncacheOneNonExistent() {
ctx := context.Background()
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
// Delete this attachment cached on disk
media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
suite.NoError(err)
suite.True(*media.Cached)
err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)
// Now attempt to uncache remote for item with db entry no file
totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false)
suite.NoError(err)
suite.Equal(2, totalUncached)
}
func TestPruneOrphanedTestSuite(t *testing.T) {
suite.Run(t, &PruneTestSuite{})
}