gotosocial/internal/federation/dereferencing/media_test.go
kim 5318054808
[performance] media processing improvements (#1288)
* media processor consolidation and reformatting, reduce amount of required syscalls

Signed-off-by: kim <grufwub@gmail.com>

* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix duration not being set, fix mp4 test expecting error

Signed-off-by: kim <grufwub@gmail.com>

* fix test expecting media files with different extension

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* fix expected storage paths in tests, update expected test thumbnails

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* fix cached presigned s3 url fetching

Signed-off-by: kim <grufwub@gmail.com>

* fix tests

Signed-off-by: kim <grufwub@gmail.com>

* fix test models

Signed-off-by: kim <grufwub@gmail.com>

* update media processing to use sync.Once{} for concurrency protection

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* fix passing in KVStore GetStream() as stream to PutStream()

Signed-off-by: kim <grufwub@gmail.com>

* fix unlocks of storage keys

Signed-off-by: kim <grufwub@gmail.com>

* whoops, return the error...

Signed-off-by: kim <grufwub@gmail.com>

* pour one out for tobi's code <3

Signed-off-by: kim <grufwub@gmail.com>

* add back the byte slurping code

Signed-off-by: kim <grufwub@gmail.com>

* check for both ErrUnexpectedEOF and EOF

Signed-off-by: kim <grufwub@gmail.com>

* add back links to file format header information

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
2023-01-11 12:13:13 +01:00

162 lines
6.1 KiB
Go

/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package dereferencing_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
type AttachmentTestSuite struct {
DereferencerStandardTestSuite
}
func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
ctx := context.Background()
fetchingAccount := suite.testAccounts["local_account_1"]
attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8"
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
RemoteURL: &attachmentURL,
Description: &attachmentDescription,
Blurhash: &attachmentBlurhash,
})
suite.NoError(err)
// make a blocking call to load the attachment from the in-process media
attachment, err := media.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(attachment)
suite.Equal(attachmentOwner, attachment.AccountID)
suite.Equal(attachmentStatus, attachment.StatusID)
suite.Equal(attachmentURL, attachment.RemoteURL)
suite.NotEmpty(attachment.URL)
suite.NotEmpty(attachment.Blurhash)
suite.NotEmpty(attachment.ID)
suite.NotEmpty(attachment.CreatedAt)
suite.NotEmpty(attachment.UpdatedAt)
suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
suite.Equal(2071680, attachment.FileMeta.Original.Size)
suite.Equal(1245, attachment.FileMeta.Original.Height)
suite.Equal(1664, attachment.FileMeta.Original.Width)
suite.Equal(attachmentBlurhash, attachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
suite.NotEmpty(attachment.File.Path)
suite.Equal(attachmentContentType, attachment.File.ContentType)
suite.Equal(attachmentDescription, attachment.Description)
suite.NotEmpty(attachment.Thumbnail.Path)
suite.NotEmpty(attachment.Type)
// attachment should also now be in the database
dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID)
suite.NoError(err)
suite.NotNil(dbAttachment)
suite.Equal(attachmentOwner, dbAttachment.AccountID)
suite.Equal(attachmentStatus, dbAttachment.StatusID)
suite.Equal(attachmentURL, dbAttachment.RemoteURL)
suite.NotEmpty(dbAttachment.URL)
suite.NotEmpty(dbAttachment.Blurhash)
suite.NotEmpty(dbAttachment.ID)
suite.NotEmpty(dbAttachment.CreatedAt)
suite.NotEmpty(dbAttachment.UpdatedAt)
suite.EqualValues(1.3365462, dbAttachment.FileMeta.Original.Aspect)
suite.Equal(2071680, dbAttachment.FileMeta.Original.Size)
suite.Equal(1245, dbAttachment.FileMeta.Original.Height)
suite.Equal(1664, dbAttachment.FileMeta.Original.Width)
suite.Equal(attachmentBlurhash, dbAttachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing)
suite.NotEmpty(dbAttachment.File.Path)
suite.Equal(attachmentContentType, dbAttachment.File.ContentType)
suite.Equal(attachmentDescription, dbAttachment.Description)
suite.NotEmpty(dbAttachment.Thumbnail.Path)
suite.NotEmpty(dbAttachment.Type)
}
func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
ctx := context.Background()
fetchingAccount := suite.testAccounts["local_account_1"]
attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8"
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
RemoteURL: &attachmentURL,
Description: &attachmentDescription,
Blurhash: &attachmentBlurhash,
})
suite.NoError(err)
attachmentID := processingMedia.AttachmentID()
time.Sleep(time.Second * 3)
// now get the attachment from the database
attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(attachment)
suite.Equal(attachmentOwner, attachment.AccountID)
suite.Equal(attachmentStatus, attachment.StatusID)
suite.Equal(attachmentURL, attachment.RemoteURL)
suite.NotEmpty(attachment.URL)
suite.NotEmpty(attachment.Blurhash)
suite.NotEmpty(attachment.ID)
suite.NotEmpty(attachment.CreatedAt)
suite.NotEmpty(attachment.UpdatedAt)
suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
suite.Equal(2071680, attachment.FileMeta.Original.Size)
suite.Equal(1245, attachment.FileMeta.Original.Height)
suite.Equal(1664, attachment.FileMeta.Original.Width)
suite.Equal(attachmentBlurhash, attachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
suite.NotEmpty(attachment.File.Path)
suite.Equal(attachmentContentType, attachment.File.ContentType)
suite.Equal(attachmentDescription, attachment.Description)
suite.NotEmpty(attachment.Thumbnail.Path)
suite.NotEmpty(attachment.Type)
}
func TestAttachmentTestSuite(t *testing.T) {
suite.Run(t, new(AttachmentTestSuite))
}