forked from mirrors/gotosocial
[feature] allow uncaching of other media types (#1234)
* simplify pruneRemote, remove unncecessary media trace logging, update RemoteOlderThan() to include headers/avis Signed-off-by: kim <grufwub@gmail.com> * cleanup pruneallmeta, add remote header to pruneremote tests Signed-off-by: kim <grufwub@gmail.com> * fix olderthan duration additions Signed-off-by: kim <grufwub@gmail.com> * fix broken test now that test model header changed Signed-off-by: kim <grufwub@gmail.com> * instead use new remote test account for new header model Signed-off-by: kim <grufwub@gmail.com> * use newer generated ULID for remote_account_3 to ensure it is sorted last Signed-off-by: kim <grufwub@gmail.com> * reorganize serialized keys to match expected test account model order Signed-off-by: kim <grufwub@gmail.com> Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
a7e71d724c
commit
58c87bdd7f
10 changed files with 192 additions and 108 deletions
|
|
@ -57,8 +57,6 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
|
|||
NewSelect().
|
||||
Model(&attachments).
|
||||
Where("? = ?", bun.Ident("media_attachment.cached"), true).
|
||||
Where("? = ?", bun.Ident("media_attachment.avatar"), false).
|
||||
Where("? = ?", bun.Ident("media_attachment.header"), false).
|
||||
Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan).
|
||||
WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")).
|
||||
Order("media_attachment.created_at DESC")
|
||||
|
|
@ -70,6 +68,7 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
|
|||
if err := q.Scan(ctx); err != nil {
|
||||
return nil, m.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
return attachments, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() {
|
|||
func (suite *MediaTestSuite) TestGetOlder() {
|
||||
attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
|
||||
suite.NoError(err)
|
||||
suite.Len(attachments, 2)
|
||||
suite.Len(attachments, 3)
|
||||
}
|
||||
|
||||
func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
|
||||
|
|
@ -49,7 +49,7 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
|
|||
|
||||
attachments, err := suite.db.GetAvatarsAndHeaders(ctx, "", 20)
|
||||
suite.NoError(err)
|
||||
suite.Len(attachments, 2)
|
||||
suite.Len(attachments, 3)
|
||||
}
|
||||
|
||||
func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() {
|
||||
|
|
|
|||
|
|
@ -29,18 +29,20 @@ import (
|
|||
type Media interface {
|
||||
// GetAttachmentByID gets a single attachment by its ID
|
||||
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
|
||||
// GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time.
|
||||
// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
|
||||
|
||||
// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
|
||||
// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
|
||||
//
|
||||
// The selected media attachments will be those with both a URL and a RemoteURL filled in.
|
||||
// In other words, media attachments that originated remotely, and that we currently have cached locally.
|
||||
GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
|
||||
|
||||
// GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers
|
||||
// and avis may be in use or not; the caller should check this if it's important.
|
||||
GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
|
||||
// GetLocalUnattachedOlderThan fetches limit n local media attachments, older than the given time, which
|
||||
// aren't header or avatars, and aren't attached to a status. In other words, attachments which were uploaded
|
||||
// but never used for whatever reason, or attachments that were attached to a status which was subsequently
|
||||
// deleted.
|
||||
|
||||
// GetLocalUnattachedOlderThan fetches limit n local media attachments (including avatars and headers), older than
|
||||
// the given time, which aren't header or avatars, and aren't attached to a status. In other words, attachments which were
|
||||
// uploaded but never used for whatever reason, or attachments that were attached to a status which was subsequently deleted.
|
||||
GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,10 +81,8 @@ func (p *ProcessingMedia) AttachmentID() string {
|
|||
// LoadAttachment blocks until the thumbnail and fullsize content
|
||||
// has been processed, and then returns the completed attachment.
|
||||
func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
|
||||
log.Tracef("LoadAttachment: getting lock for attachment %s", p.attachment.URL)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
log.Tracef("LoadAttachment: got lock for attachment %s", p.attachment.URL)
|
||||
|
||||
if err := p.store(ctx); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -98,23 +96,24 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// store the result in the database before returning it
|
||||
if !p.insertedInDB {
|
||||
if p.recache {
|
||||
// if it's a recache we should only need to update
|
||||
// This is an existing media attachment we're recaching, so only need to update it
|
||||
if err := p.database.UpdateByID(ctx, p.attachment, p.attachment.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// otherwise we need to really PUT it
|
||||
// This is a new media attachment we're caching for first time
|
||||
if err := p.database.Put(ctx, p.attachment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Mark this as stored in DB
|
||||
p.insertedInDB = true
|
||||
}
|
||||
|
||||
log.Tracef("LoadAttachment: finished, returning attachment %s", p.attachment.URL)
|
||||
log.Tracef("finished loading attachment %s", p.attachment.URL)
|
||||
return p.attachment, nil
|
||||
}
|
||||
|
||||
|
|
@ -180,7 +179,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
|
|||
|
||||
// we're done processing the thumbnail!
|
||||
atomic.StoreInt32(&p.thumbState, int32(complete))
|
||||
log.Tracef("loadThumb: finished processing thumbnail for attachment %s", p.attachment.URL)
|
||||
log.Tracef("finished processing thumbnail for attachment %s", p.attachment.URL)
|
||||
fallthrough
|
||||
case complete:
|
||||
return nil
|
||||
|
|
@ -241,7 +240,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
|
|||
|
||||
// we're done processing the full-size image
|
||||
atomic.StoreInt32(&p.fullSizeState, int32(complete))
|
||||
log.Tracef("loadFullSize: finished processing full size image for attachment %s", p.attachment.URL)
|
||||
log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
|
||||
fallthrough
|
||||
case complete:
|
||||
return nil
|
||||
|
|
@ -362,7 +361,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
|
|||
p.attachment.File.FileSize = int(fileSize)
|
||||
p.read = true
|
||||
|
||||
log.Tracef("store: finished storing initial data for attachment %s", p.attachment.URL)
|
||||
log.Tracef("finished storing initial data for attachment %s", p.attachment.URL)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package media
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"codeberg.org/gruf/go-store/v2/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
|
|
@ -28,17 +29,23 @@ import (
|
|||
)
|
||||
|
||||
func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
|
||||
var totalPruned int
|
||||
var maxID string
|
||||
var attachments []*gtsmodel.MediaAttachment
|
||||
var err error
|
||||
var (
|
||||
totalPruned int
|
||||
maxID string
|
||||
)
|
||||
|
||||
for {
|
||||
// select "selectPruneLimit" headers / avatars at a time for pruning
|
||||
attachments, err := m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit)
|
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
return totalPruned, err
|
||||
} else if len(attachments) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// select 20 attachments at a time and prune them
|
||||
for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) {
|
||||
// use the id of the last attachment in the slice as the next 'maxID' value
|
||||
l := len(attachments)
|
||||
log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID)
|
||||
maxID = attachments[l-1].ID
|
||||
log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", len(attachments), maxID)
|
||||
maxID = attachments[len(attachments)-1].ID
|
||||
|
||||
// prune each attachment that meets one of the following criteria:
|
||||
// - has no owning account in the database
|
||||
|
|
@ -56,11 +63,6 @@ func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// make sure we don't have a real error when we leave the loop
|
||||
if err != nil && err != db.ErrNoEntries {
|
||||
return totalPruned, err
|
||||
}
|
||||
|
||||
log.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned)
|
||||
return totalPruned, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ package media
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-store/v2/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
|
|
@ -31,21 +32,23 @@ import (
|
|||
func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) {
|
||||
var totalPruned int
|
||||
|
||||
olderThan, err := parseOlderThan(olderThanDays)
|
||||
if err != nil {
|
||||
return totalPruned, fmt.Errorf("PruneAllRemote: error parsing olderThanDays %d: %s", olderThanDays, err)
|
||||
}
|
||||
olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays))
|
||||
log.Infof("PruneAllRemote: pruning media older than %s", olderThan)
|
||||
|
||||
// select 20 attachments at a time and prune them
|
||||
for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
|
||||
for {
|
||||
// Select "selectPruneLimit" status attacchments at a time for pruning
|
||||
attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit)
|
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
return totalPruned, err
|
||||
} else if len(attachments) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
|
||||
l := len(attachments)
|
||||
log.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan)
|
||||
olderThan = attachments[l-1].CreatedAt
|
||||
// use the age of the oldest attachment (last in slice) as the next 'olderThan' value
|
||||
log.Tracef("PruneAllRemote: got %d status attachments older than %s", len(attachments), olderThan)
|
||||
olderThan = attachments[len(attachments)-1].CreatedAt
|
||||
|
||||
// prune each attachment
|
||||
// prune each status attachment
|
||||
for _, attachment := range attachments {
|
||||
if err := m.pruneOneRemote(ctx, attachment); err != nil {
|
||||
return totalPruned, err
|
||||
|
|
@ -54,11 +57,6 @@ func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, e
|
|||
}
|
||||
}
|
||||
|
||||
// make sure we don't have a real error when we leave the loop
|
||||
if err != nil && err != db.ErrNoEntries {
|
||||
return totalPruned, err
|
||||
}
|
||||
|
||||
log.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned)
|
||||
return totalPruned, nil
|
||||
}
|
||||
|
|
@ -69,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
|
|||
if attachment.File.Path != "" {
|
||||
// delete the full size attachment from storage
|
||||
log.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
|
||||
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
|
||||
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
cached := false
|
||||
|
|
@ -80,7 +78,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
|
|||
if attachment.Thumbnail.Path != "" {
|
||||
// delete the thumbnail from storage
|
||||
log.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
|
||||
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
|
||||
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
cached := false
|
||||
|
|
@ -88,10 +86,10 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
|
|||
changed = true
|
||||
}
|
||||
|
||||
// update the attachment to reflect that we no longer have it cached
|
||||
if changed {
|
||||
return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
// update the attachment to reflect that we no longer have it cached
|
||||
return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
|
||||
"codeberg.org/gruf/go-store/v2/storage"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
)
|
||||
|
||||
type PruneRemoteTestSuite struct {
|
||||
|
|
@ -34,24 +35,29 @@ type PruneRemoteTestSuite struct {
|
|||
}
|
||||
|
||||
func (suite *PruneRemoteTestSuite) TestPruneRemote() {
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
suite.True(*testAttachment.Cached)
|
||||
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
suite.True(*testStatusAttachment.Cached)
|
||||
|
||||
testHeader := suite.testAttachments["remote_account_3_header"]
|
||||
suite.True(*testHeader.Cached)
|
||||
|
||||
totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
|
||||
suite.NoError(err)
|
||||
suite.Equal(2, totalPruned)
|
||||
suite.Equal(3, totalPruned)
|
||||
|
||||
prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
|
||||
prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID)
|
||||
suite.NoError(err)
|
||||
suite.False(*prunedAttachment.Cached)
|
||||
|
||||
// the media should no longer be cached
|
||||
prunedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID)
|
||||
suite.NoError(err)
|
||||
suite.False(*prunedAttachment.Cached)
|
||||
}
|
||||
|
||||
func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
|
||||
totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
|
||||
suite.NoError(err)
|
||||
suite.Equal(2, totalPruned)
|
||||
suite.Equal(3, totalPruned)
|
||||
|
||||
// final prune should prune nothing, since the first prune already happened
|
||||
totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1)
|
||||
|
|
@ -61,16 +67,21 @@ func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
|
|||
|
||||
func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
|
||||
ctx := context.Background()
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
testHeader := suite.testAttachments["remote_account_3_header"]
|
||||
|
||||
totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
|
||||
suite.NoError(err)
|
||||
suite.Equal(2, totalPruned)
|
||||
suite.Equal(3, totalPruned)
|
||||
|
||||
// media should no longer be stored
|
||||
_, err = suite.storage.Get(ctx, testAttachment.File.Path)
|
||||
_, err = suite.storage.Get(ctx, testStatusAttachment.File.Path)
|
||||
suite.ErrorIs(err, storage.ErrNotFound)
|
||||
_, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
|
||||
_, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path)
|
||||
suite.ErrorIs(err, storage.ErrNotFound)
|
||||
_, err = suite.storage.Get(ctx, testHeader.File.Path)
|
||||
suite.ErrorIs(err, storage.ErrNotFound)
|
||||
_, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path)
|
||||
suite.ErrorIs(err, storage.ErrNotFound)
|
||||
|
||||
// now recache the image....
|
||||
|
|
@ -82,34 +93,40 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
|
|||
}
|
||||
return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
|
||||
}
|
||||
processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID)
|
||||
suite.NoError(err)
|
||||
|
||||
// synchronously load the recached attachment
|
||||
recachedAttachment, err := processingRecache.LoadAttachment(ctx)
|
||||
suite.NoError(err)
|
||||
suite.NotNil(recachedAttachment)
|
||||
for _, original := range []*gtsmodel.MediaAttachment{
|
||||
testStatusAttachment,
|
||||
testHeader,
|
||||
} {
|
||||
processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID)
|
||||
suite.NoError(err)
|
||||
|
||||
// recachedAttachment should be basically the same as the old attachment
|
||||
suite.True(*recachedAttachment.Cached)
|
||||
suite.Equal(testAttachment.ID, recachedAttachment.ID)
|
||||
suite.Equal(testAttachment.File.Path, recachedAttachment.File.Path) // file should be stored in the same place
|
||||
suite.Equal(testAttachment.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
|
||||
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
|
||||
// synchronously load the recached attachment
|
||||
recachedAttachment, err := processingRecache.LoadAttachment(ctx)
|
||||
suite.NoError(err)
|
||||
suite.NotNil(recachedAttachment)
|
||||
|
||||
// recached files should be back in storage
|
||||
_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
// recachedAttachment should be basically the same as the old attachment
|
||||
suite.True(*recachedAttachment.Cached)
|
||||
suite.Equal(original.ID, recachedAttachment.ID)
|
||||
suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place
|
||||
suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
|
||||
suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
|
||||
|
||||
// recached files should be back in storage
|
||||
_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
|
||||
suite.NoError(err)
|
||||
_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
|
||||
suite.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
|
||||
ctx := context.Background()
|
||||
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
|
||||
|
||||
// Delete this attachment cached on disk
|
||||
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
|
||||
media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
|
||||
suite.NoError(err)
|
||||
suite.True(*media.Cached)
|
||||
err = suite.storage.Delete(ctx, media.File.Path)
|
||||
|
|
@ -118,7 +135,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
|
|||
// Now attempt to prune remote for item with db entry no file
|
||||
totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
|
||||
suite.NoError(err)
|
||||
suite.Equal(2, totalPruned)
|
||||
suite.Equal(3, totalPruned)
|
||||
}
|
||||
|
||||
func TestPruneRemoteTestSuite(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package media
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-store/v2/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
|
|
@ -34,10 +34,7 @@ func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error)
|
|||
var attachments []*gtsmodel.MediaAttachment
|
||||
var err error
|
||||
|
||||
olderThan, err := parseOlderThan(UnusedLocalAttachmentCacheDays)
|
||||
if err != nil {
|
||||
return totalPruned, fmt.Errorf("PruneUnusedLocalAttachments: error parsing olderThanDays %d: %s", UnusedLocalAttachmentCacheDays, err)
|
||||
}
|
||||
olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(UnusedLocalAttachmentCacheDays))
|
||||
log.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan)
|
||||
|
||||
// select 20 attachments at a time and prune them
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/h2non/filetype"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
|
|
@ -134,22 +133,6 @@ func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{
|
|||
log.Error("media manager cron logger: ", err, msg, keysAndValues)
|
||||
}
|
||||
|
||||
func parseOlderThan(olderThanDays int) (time.Time, error) {
|
||||
// convert days into a duration string
|
||||
olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
|
||||
|
||||
// parse the duration string into a duration
|
||||
olderThanHours, err := time.ParseDuration(olderThanHoursString)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
// 'subtract' that from the time now to give our threshold
|
||||
olderThan := time.Now().Add(-olderThanHours)
|
||||
|
||||
return olderThan, nil
|
||||
}
|
||||
|
||||
// lengthReader wraps a reader and reads the length of total bytes written as it goes.
|
||||
type lengthReader struct {
|
||||
source io.Reader
|
||||
|
|
|
|||
|
|
@ -573,6 +573,43 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
|
|||
HideCollections: FalseBool(),
|
||||
SuspensionOrigin: "",
|
||||
},
|
||||
"remote_account_3": {
|
||||
ID: "062G5WYKY35KKD12EMSM3F8PJ8",
|
||||
Username: "her_fuckin_maj",
|
||||
Domain: "thequeenisstillalive.technology",
|
||||
DisplayName: "lizzzieeeeeeeeeeee",
|
||||
Fields: []gtsmodel.Field{},
|
||||
Note: "if i die blame charles don't let that fuck become king",
|
||||
Memorial: FalseBool(),
|
||||
MovedToAccountID: "",
|
||||
CreatedAt: TimeMustParse("2020-08-10T14:13:28+02:00"),
|
||||
UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"),
|
||||
Bot: FalseBool(),
|
||||
Locked: TrueBool(),
|
||||
Discoverable: TrueBool(),
|
||||
Sensitive: FalseBool(),
|
||||
Language: "en",
|
||||
URI: "http://thequeenisstillalive.technology/users/her_fuckin_maj",
|
||||
URL: "http://thequeenisstillalive.technology/@her_fuckin_maj",
|
||||
LastWebfingeredAt: time.Time{},
|
||||
InboxURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/inbox",
|
||||
SharedInboxURI: StringPtr(""),
|
||||
OutboxURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/outbox",
|
||||
FollowersURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/followers",
|
||||
FollowingURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/following",
|
||||
FeaturedCollectionURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/collections/featured",
|
||||
ActorType: ap.ActorPerson,
|
||||
AlsoKnownAs: "",
|
||||
PrivateKey: &rsa.PrivateKey{},
|
||||
PublicKey: &rsa.PublicKey{},
|
||||
PublicKeyURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj#main-key",
|
||||
SensitizedAt: time.Time{},
|
||||
SilencedAt: time.Time{},
|
||||
SuspendedAt: time.Time{},
|
||||
HideCollections: FalseBool(),
|
||||
SuspensionOrigin: "",
|
||||
HeaderMediaAttachmentID: "01PFPMWK2FF0D9WMHEJHR07C3R",
|
||||
},
|
||||
}
|
||||
|
||||
var accountsSorted []*gtsmodel.Account
|
||||
|
|
@ -585,6 +622,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
|
|||
})
|
||||
|
||||
preserializedKeys := []string{
|
||||
"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA3bAoQMUofndXMXikEU2MOJbfI1uaZbIDrxW0bEO6IOhwe/J0jWJHL2fWc2mbp2NxAH4Db1kZIcl9D0owoRf2cT5k0Y2Dah86dGz4fIedkqGryoWAnEJ2hHKGXGQf2K9OS2L8eaDLGU4CBds0m80vrn153Uiyj7zxWDYqcySM0qQjSg+mvgqpBcxKpd+xACaWNDL8qWvDsBF1D0RuO8hUiXMIKOUoFAGbqe6qWGK0COrEYQTAMydoFuSaAccP70zKQslnSOCKvsOi/iPRKGDNqWINIC/lwqXEIpMj3K+b/A+x41zR7frTgHNLbe4yHWAVNPEwTFningbB/lIyyVmDAgMBAAECggEBALxwnipmRnyvPClMY+RiJ5PGwtqYcGsly82/pwRW98GHX7Rv1lA8x/ZnghxNPbVg0k9ZvMXcaICeu4BejQ2AiKo4sU7OVGc/K+3wTXxoKBU0bJQuV0x24JVuCXvwD7/x9i8Yh0nKCOoH+mkNkcUQKWXaJi0IoXwd5u0kVCAbym1vux/9DcwtydqT4P1EoxEHCXDuRorBP8vYWCZBwRY2etmdAEbHsVpVlNlXWfbGCNMf5e8AecOZre4No8UfTOZkM7YKgjryde3YCmY2zDQI9jExGD2L5nptLizODD5imdpp/IQ7qg6rR3XbIK6CDiKiePEFQibD8XWiz7XVD6JBRokCgYEA0jEAxZseHUyobh1ERHezs2vC2zbiTOfnOpFxhwtNt67dUQZDssTxXF+BymUL8yKi1bnheOTuyASxrgZ7BPdiFvJfhlelSxtxtt1RamY58E179uiel2NPRsR3SL2AsGg+jP+QjJpsJHvYIliXP38G7NVaqaSMFgXfXir7Ty7W0r0CgYEA6uYQWfjmaB66xPrL/oCBaJ+UWM/Zdfw4IETVnRVOxVqGE7AKqC+31fZQ5kIXnNcJNLJ0OJlhGH5vZYp/r4z6qly9BUVolCJcW2YLEOOnChOvKGwlDSXrdGty2f34RXdABwsf/pBHsdpJq70+SE01tTB/8P2NTnRafy9GL/FnwT8CgYEAjJ4D6i8wImHafHBP7441Rl9daNJ66wBqDSCoVrQVNkFiBoauW7at0iKC7ihTqkENtvY4BW0C4gVh6Q6k1lm54agch/+ysWCW3sOJaCkjscPknvZYwubJboqZUqyUn2/eCO4ggi/9ERtZKQEjjnMo6uCBWuSeY01iddlDb2HijfECgYBYQCM4ikiWKaVlyAvIDCOSWRH04/IBX8b+aJ4QrCayAraIwwTd9z+MBUSTnZUdebSdtcXwVb+i4i2b6pLaM48hXkItrswBi39DX20c5UqmgIq4Fxk8fVienpfByqbyAkFt5AIbM72b1jUDbs/tfgSFlDkdI0VpilFNo0ctT/b5JQKBgAxPGtVGzhSQUZWPXjhiBT7MM/1EiLBYhGVrymzd9dmBxj+UyifnRXfIQbOQm3EfI5Z8ZpyS6eqWdi9NTeZi8rg0WleMb/VbOMT3xvTO34vDXvwrQKhFMimX1tY7aKy1udnE2ON2/alq2zWo3zPZfYH1KFdDtGD08GW2M4OO1caa",
|
||||
"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGj2wLnDIHnP6wjJ+WmIhp7NGAaKWwfxBWfdMFR+Y0ilkK5ld5igT45UHAmzN3v4HcwHGGpPITD9caDYj5YaGOX+dSdGLgXWwItR0j+ivrHEJmvz8hG6z9wKEZKUUrRw7Ob72S0LOsreq98bjdiWJKHNka27slqQjGyhLQtcg6pe1CLJtnuJH4GEMLj7jJB3/Mqv3vl5CQZ+Js0bXfgw5TF/x/Bzq/8qsxQ1vnmYHJsR0eLPEuDJOvoFPiJZytI09S7qBEJL5PDeVSfjQi3o71sqOzZlEL0b0Ny48rfo/mwJAdkmfcnydRDxeGUEqpAWICCOdUL0+W3/fCffaRZsk1AgMBAAECggEAUuyO6QJgeoF8dGsmMxSc0/ANRp1tpRpLznNZ77ipUYP9z+mG2sFjdjb4kOHASuB18aWFRAAbAQ76fGzuqYe2muk+iFcG/EDH35MUCnRuZxA0QwjX6pHOW2NZZFKyCnLwohJUj74Na65ufMk4tXysydrmaKsfq4i+m5bE6NkiOCtbXsjUGVdJKzkT6X1gEyEPEHgrgVZz9OpRY5nwjZBMcFI6EibFnWdehcuCQLESIX9ll/QzGvTJ1p8xeVJs2ktLWKQ38RewwucNYVLVJmxS1LCPP8x+yHVkOxD66eIncY26sjX+VbyICkaG/ZjKBuoOekOq/T+b6q5ESxWUNfcu+QKBgQDmt3WVBrW6EXKtN1MrVyBoSfn9WHyf8Rfb84t5iNtaWGSyPZK/arUw1DRbI0TdPjct//wMWoUU2/uqcPSzudTaPena3oxjKReXso1hcynHqboCaXJMxWSqDQLumbrVY05C1WFSyhRY0iQS5fIrNzD4+6rmeC2Aj5DKNW5Atda8dwKBgQDcUdhQfjL9SmzzIeAqJUBIfSSI2pSTsZrnrvMtSMkYJbzwYrUdhIVxaS4hXuQYmGgwonLctyvJxVxEMnf+U0nqPgJHE9nGQb5BbK6/LqxBWRJQlc+W6EYodIwvtE5B4JNkPE5757u+xlDdHe2zGUGXSIf4IjBNbSpCu6RcFsGOswKBgEnr4gqbmcJCMOH65fTu930yppxbq6J7Vs+sWrXX+aAazjilrc0S3XcFprjEth3E/10HtbQnlJg4W4wioOSs19wNFk6AG67xzZNXLCFbCrnkUarQKkUawcQSYywbqVcReFPFlmc2RAqpWdGMR2k9R72etQUe4EVeul9veyHUoTbFAoGBAKj3J9NLhaVVb8ri3vzThsJRHzTJlYrTeb5XIO5I1NhtEMK2oLobiQ+aH6O+F2Z5c+Zgn4CABdf/QSyYHAhzLcu0dKC4K5rtjpC0XiwHClovimk9C3BrgGrEP0LSn/XL2p3T1kkWRpkflKKPsl1ZcEEqggSdi7fFkdSN/ZYWaakbAoGBALWVGpA/vXmaZEV/hTDdtDnIHj6RXfKHCsfnyI7AdjUX4gokzdcEvFsEIoI+nnXR/PIAvwqvQw4wiUqQnp2VB8r73YZvW/0npnsidQw3ZjqnyvZ9X8y80nYs7DjSlaG0A8huy2TUdFnJyCMWby30g82kf0b/lhotJg4d3fIDou51",
|
||||
"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6q61hiC7OhlMz7JNnLiL/RwOaFC8955GDvwSMH9Zw3oguWH9nLqkmlJ98cnqRG9ZC0qVo6Gagl7gv6yOHDwD4xZI8JoV2ZfNdDzq4QzoBIzMtRsbSS4IvrF3JP+kDH1tim+CbRMBxiFJgLgS6yeeQlLNvBW+CIYzmeCimZ6CWCr91rZPIprUIdjvhxrM9EQU072Pmzn2gpGM6K5gAReN+LtP+VSBC61x7GQJxBaJNtk11PXkgG99EdFi9vvgEBbM9bdcawvf8jxvjgsgdaDx/1cypDdnaL8eistmyv1YI67bKvrSPCEh55b90hl3o3vW4W5G4gcABoyORON96Y+i9AgMBAAECggEBAKp+tyNH0QiMo13fjFpHR2vFnsKSAPwXj063nx2kzqXUeqlp5yOE+LXmNSzjGpOCy1XJM474BRRUvsP1jkODLq4JNiF+RZP4Vij/CfDWZho33jxSUrIsiUGluxtfJiHV+A++s4zdZK/NhP+XyHYah0gEqUaTvl8q6Zhu0yH5sDCZHDLxDBpgiT5qD3lli8/o2xzzBdaibZdjQyHi9v5Yi3+ysly1tmfmqnkXSsevAubwJu504WxvDUSo7hPpG4a8Xb8ODqL738GIF2UY/olCcGkWqTQEr2pOqG9XbMmlUWnxG62GCfK6KtGfIzCyBBkGO2PZa9aPhVnv2bkYxI4PkLkCgYEAzAp7xH88UbSX31suDRa4jZwgtzhJLeyc3YxO5C4XyWZ89oWrA30V1KvfVwFRavYRJW07a+r0moba+0E1Nj5yZVXPOVu0bWd9ZyMbdH2L6MRZoJWU5bUOwyruulRCkqASZbWo4G05NOVesOyY1bhZGE7RyUW0vOo8tSyyRQ8nUGMCgYEA6jTQbDry4QkUP9tDhvc8+LsobIF1mPLEJui+mT98+9IGar6oeVDKekmNDO0Dx2+miLfjMNhCb5qUc8g036ZsekHt2WuQKunADua0coB00CebMdr6AQFf7QOQ/RuA+/gPJ5G0GzWB3YOQ5gE88tTCO/jBfmikVOZvLtgXUGjo3F8CgYEAl2poMoehQZjc41mMsRXdWukztgPE+pmORzKqENbLvB+cOG01XV9j5fCtyqklvFRioP2QjSNM5aeRtcbMMDbjOaQWJaCSImYcP39kDmxkeRXM1UhruJNGIzsm8Ys55Al53ZSTgAhN3Z0hSfYp7N/i7hD/yXc7Cr5g0qoamPkH2bUCgYApf0oeoyM9tDoeRl9knpHzEFZNQ3LusrUGn96FkLY4eDIi371CIYp+uGGBlM1CnQnI16wtj2PWGnGLQkH8DqTR1LSr/V8B+4DIIyB92TzZVOsunjoFy5SPjj42WpU0D/O/cxWSbJyh/xnBZx7Bd+kibyT5nNjhIiM5DZiz6qK3yQKBgAOO/MFKHKpKOXrtafbqCyculG/ope2u4eBveHKO6ByWcUSbuD9ebtr7Lu5AC5tKUJLkSyRx4EHk71bqP1yOITj8z9wQWdVyLxtVtyj9SUkUNvGwIj+F7NJ5VgHzWVZtvYWDCzrfxkEhKk3DRIIVjqmEohJcaOZoZ2Q/f8sjlId6",
|
||||
"MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1NzommDoutE+FVAbgovPb5ioRRS1k93hhH5Mpe4KfAQb1k0aGA/TjrFr2HcRbOtldo6Fe+RRCAm5Go+sBx829zyMEdXbGtR4Pym78xYoRpsCwD+2AK/edQLjsdDf9zXZod9ig/Pe59awYaeuSFyK/w9q94ncuiE7m+MKfXJnTS/qiwwkxWIRm9lBprPIT0DwXCtoh7FdpsOmjLu2QdGADV+9KSDgV5IbVcxwjPY03vHJS4UAIP5eS46TtSrNF3hyM9Q8vGIPAixOVyAY53cRQUxZWU/FIaNjaEgpreUQfK1pgr0gxh1K7IKwmyF3f/JgL0urFljYp2UonzRU5XKHJAgMBAAECggEBAKVT2HLDqTlY+b/LNGcXY+H4b+LHuS2HdUUuuGU9MKN+HWpIziuQSoi4g1hNOgp9ezgqBByQpAHBE/jQraQ3NKZ55xm3TQDm1qFTb8SfOGL4Po2iSm0IL+VA2jWnpjmgjOmshXACusPmtfakE+55uxM3TUa16UQDyfCBfZZEtnaFLTYzJ7KmD2GPot8SCxJBqNmW7AL8pMSIxMC3cRxUbK4R3+KIisXUuB50jZH3zGHxi34e2jA6gDeFmzgHCDJRidHMsCTHTaATzlvVz9YwwNqPQaYY7OFouZXwFxVAxIg/1zVvLc3zx1gWt+UDFeI7h6Eq0h5DZPdUiR4mrhAKd70CgYEAw6WKbPgjzhJI9XVmnu0aMHHH4MK8pbIq4kddChw24yZv1e9qnNTHw3YK17X9Fqog9CU1OX3M/vddfQbc34SorBmtmGYgOfDSuXTct52Ppyl4CRwndYQc0A88Hw+klluTEPY3+NRV6YSzv8vkNMasVuOh0YI1xzbpc+Bb5LL3kwMCgYEA7R4PLYYmtzKAY2YTQOXGBh3xd6UEHgks30W+QzDxvOv75svZt6yDgiwJzXtyrQzbNaH6yca5nfjkqyhnHwpguJ6DK7+S/RnZfVib5MqRwiU7g8l3neKhIXs6xZxfORunDU9T5ntbyNaGv/TJ2cXNw+9VskhBaHfEN/kmaBNNuEMCgYARLuzlfTXH15tI07Lbqn9uWc/wUao381oI3bOyO6Amey2/YHPAqn+RD0EMiRNddjvGta3jCsWCbz9qx7uGdiRKWUcB55ZVAG3BlB3+knwXdnDwe+SLUbsmGvBw2fLesdRM3RM1a5DQHbOb2NCGQhzI1N1VhVYr1QrT/pSTlZRg+QKBgCE05nc/pEhfoC9LakLaauMManaQ+4ShUFFsWPrb7d7BRaPKxJC+biRauny2XxbxB/n410BOvkvrQUre+6ITN/xi5ofH6nPbnOO69woRfFwuDqmkG0ZXKK2hrldiUMuUnc51X5CVkgMMWA6l32bKFsjryZqQF+jjbO1RzRkiKu41AoGAHQer1NyajHEpEfempx8YTsAnOn+Hi33cXAaQoTkS41lX2YK0cBkD18yhubczZcKnMW+GRKZRYXMm0NfwiuIo5oIYWeO6K+rXF+SKptC5mnw/3FhDVnghDAmEqOcRSWnFXARk1WEbFtwG5phDeFrWXsqPzGAjoZ8bhLvKRsrG4OM=",
|
||||
|
|
@ -979,6 +1017,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
|
|||
Header: FalseBool(),
|
||||
Cached: TrueBool(),
|
||||
},
|
||||
"remote_account_3_header": {
|
||||
ID: "01PFPMWK2FF0D9WMHEJHR07C3R",
|
||||
StatusID: "",
|
||||
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
|
||||
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
|
||||
CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
|
||||
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
|
||||
Type: gtsmodel.FileTypeImage,
|
||||
FileMeta: gtsmodel.FileMeta{
|
||||
Original: gtsmodel.Original{
|
||||
Width: 472,
|
||||
Height: 291,
|
||||
Size: 137352,
|
||||
Aspect: 1.6219931271477663,
|
||||
},
|
||||
Small: gtsmodel.Small{
|
||||
Width: 472,
|
||||
Height: 291,
|
||||
Size: 137352,
|
||||
Aspect: 1.6219931271477663,
|
||||
},
|
||||
Focus: gtsmodel.Focus{
|
||||
X: 0,
|
||||
Y: 0,
|
||||
},
|
||||
},
|
||||
AccountID: "062G5WYKY35KKD12EMSM3F8PJ8",
|
||||
Description: "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
|
||||
ScheduledStatusID: "",
|
||||
Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
|
||||
Processing: 2,
|
||||
File: gtsmodel.File{
|
||||
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
|
||||
ContentType: "image/jpeg",
|
||||
FileSize: 19310,
|
||||
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
|
||||
},
|
||||
Thumbnail: gtsmodel.Thumbnail{
|
||||
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
|
||||
ContentType: "image/jpeg",
|
||||
FileSize: 20395,
|
||||
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
|
||||
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
|
||||
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
|
||||
},
|
||||
Avatar: FalseBool(),
|
||||
Header: TrueBool(),
|
||||
Cached: TrueBool(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue