mirrors/gotosocial
1
0
Fork 1
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2024-06-21 14:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

rework data function to provide filesize

Browse source
This commit is contained in:
tsmethurst 2022-01-23 14:41:58 +01:00
parent 7d024ce74d
commit c157b1b20b
12 changed files with 56 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
internal/federation/dereferencing/account.go
View file

@ -252,7 +252,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount *
return err
}
data := func(innerCtx context.Context) (io.Reader, error) {
data := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, avatarIRI)
}
@ -274,7 +274,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount *
return err
}
data := func(innerCtx context.Context) (io.Reader, error) {
data := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, headerIRI)
}

2
internal/federation/dereferencing/media.go
View file

@ -42,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a
return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err)
}
dataFunc := func(innerCtx context.Context) (io.Reader, error) {
dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, derefURI)
}

20
internal/media/image.go
View file

@ -30,7 +30,6 @@ import (
"github.com/buckket/go-blurhash"
"github.com/nfnt/resize"
"github.com/superseriousbusiness/exifremove/pkg/exifremove"
)
const (
@ -197,22 +196,3 @@ func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) {
small: out.Bytes(),
}, nil
}
// purgeExif is a little wrapper for the action of removing exif data from an image.
// Only pass pngs or jpegs to this function.
func purgeExif(data []byte) ([]byte, error) {
if len(data) == 0 {
return nil, errors.New("passed image was not valid")
}
clean, err := exifremove.Remove(data)
if err != nil {
return nil, fmt.Errorf("could not purge exif from image: %s", err)
}
if len(clean) == 0 {
return nil, errors.New("purged image was not valid")
}
return clean, nil
}

12
internal/media/manager_test.go
View file

@ -39,13 +39,13 @@ type ManagerTestSuite struct {
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, error) {
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), nil
return bytes.NewBuffer(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -109,13 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, error) {
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), nil
return bytes.NewBuffer(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -192,9 +192,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
panic(err)
}
data := func(_ context.Context) (io.Reader, error) {
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
return bytes.NewReader(b), nil
return bytes.NewReader(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"

4
internal/media/processingemoji.go
View file

@ -163,7 +163,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
}
// execute the data function to get the reader out of it
reader, err := p.data(ctx)
reader, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
@ -194,6 +194,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension)
p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension)
p.emoji.ImageContentType = contentType
p.emoji.ImageFileSize = fileSize
// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
@ -202,7 +203,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream
// if the original reader is a readcloser, close it since we're done with it now
if rc, ok := reader.(io.ReadCloser); ok {

33
internal/media/processingmedia.go
View file

@ -28,6 +28,7 @@ import (
"time"
"codeberg.org/gruf/go-store/kv"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
@ -239,7 +240,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
}
// execute the data function to get the reader out of it
reader, err := p.data(ctx)
reader, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
@ -268,22 +269,36 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
}
extension := split[1] // something like 'jpeg'
// set some additional fields on the attachment now that
// we know more about what the underlying media actually is
if extension == mimeGif {
// concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// we'll need to clean exif data from the first bytes; while we're
// here, we can also use the extension to derive the attachment type
var clean io.Reader
switch extension {
case mimeGif:
p.attachment.Type = gtsmodel.FileTypeGif
} else {
clean = multiReader // nothing to clean from a gif
case mimeJpeg, mimePng:
p.attachment.Type = gtsmodel.FileTypeImage
purged, err := terminator.Terminate(multiReader, fileSize, extension)
if err != nil {
return fmt.Errorf("store: exif error: %s", err)
}
clean = purged
default:
return fmt.Errorf("store: couldn't process %s", extension)
}
// now set some additional fields on the attachment since
// we know more about what the underlying media actually is
p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension)
p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension)
p.attachment.File.ContentType = contentType
// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
p.attachment.File.FileSize = fileSize
// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil {
if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}

2
internal/media/types.go
View file

@ -118,4 +118,4 @@ type AdditionalEmojiInfo struct {
}
// DataFunc represents a function used to retrieve the raw bytes of a piece of media.
type DataFunc func(ctx context.Context) (io.Reader, error)
type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error)

10
internal/processing/account/update.go
View file

@ -140,8 +140,9 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead
return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
}
dataFunc := func(ctx context.Context) (io.Reader, error) {
return avatar.Open()
dataFunc := func(ctx context.Context) (io.Reader, int, error) {
f, err := avatar.Open()
return f, int(avatar.Size), err
}
isAvatar := true
@ -166,8 +167,9 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead
return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
}
dataFunc := func(ctx context.Context) (io.Reader, error) {
return header.Open()
dataFunc := func(ctx context.Context) (io.Reader, int, error) {
f, err := header.Open()
return f, int(header.Size), err
}
isHeader := true

5
internal/processing/admin/emoji.go
View file

@ -36,8 +36,9 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
data := func(innerCtx context.Context) (io.Reader, error) {
return form.Image.Open()
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.Image.Open()
return f, int(form.Image.Size), err
}
emojiID, err := id.NewRandomULID()

5
internal/processing/media/create.go
View file

@ -29,8 +29,9 @@ import (
)
func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) {
data := func(innerCtx context.Context) (io.Reader, error) {
return form.File.Open()
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.File.Open()
return f, int(form.File.Size), err
}
focusX, focusY, err := parseFocus(form.Focus)

12
internal/transport/derefmedia.go
View file

@ -28,12 +28,12 @@ import (
"github.com/sirupsen/logrus"
)
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) {
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) {
l := logrus.WithField("func", "DereferenceMedia")
l.Debugf("performing GET to %s", iri.String())
req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil)
if err != nil {
return nil, err
return nil, 0, err
}
req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here
@ -44,14 +44,14 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.Read
err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
t.getSignerMu.Unlock()
if err != nil {
return nil, err
return nil, 0, err
}
resp, err := t.client.Do(req)
if err != nil {
return nil, err
return nil, 0, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
return nil, 0, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
}
return resp.Body, nil
return resp.Body, int(resp.ContentLength), nil
}

4
internal/transport/transport.go
View file

@ -34,8 +34,8 @@ import (
// functionality for fetching remote media.
type Transport interface {
pub.Transport
// DereferenceMedia fetches the given media attachment IRI.
DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error)
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error)
// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.