diff --git a/go.mod b/go.mod
index 4ff8c87a1..135dc2990 100644
--- a/go.mod
+++ b/go.mod
@@ -46,7 +46,7 @@ require (
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.8.4
- github.com/superseriousbusiness/activity v1.6.0-gts
+ github.com/superseriousbusiness/activity v1.6.0-gts.0.20240221151241-5d56c04088d4
github.com/superseriousbusiness/httpsig v1.2.0-SSB
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8
github.com/tdewolff/minify/v2 v2.20.17
diff --git a/go.sum b/go.sum
index 57fde77d9..e3ac40382 100644
--- a/go.sum
+++ b/go.sum
@@ -483,8 +483,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I=
-github.com/superseriousbusiness/activity v1.6.0-gts h1:SwrTpqof0bIzYYsNyM7WH9Vxqz+6kN4BGQjzKvlIN1Y=
-github.com/superseriousbusiness/activity v1.6.0-gts/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM=
+github.com/superseriousbusiness/activity v1.6.0-gts.0.20240221151241-5d56c04088d4 h1:kPjQR/hVZtROTzkxptp/EIR7Wm58O8jppwpCFrZ7sVU=
+github.com/superseriousbusiness/activity v1.6.0-gts.0.20240221151241-5d56c04088d4/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM=
github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe h1:ksl2oCx/Qo8sNDc3Grb8WGKBM9nkvhCm25uvlT86azE=
github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe/go.mod h1:gH4P6gN1V+wmIw5o97KGaa1RgXB/tVpC2UNzijhg3E4=
github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB h1:8psprYSK1KdOSH7yQ4PbJq0YYaGQY+gzdW/B0ExDb/8=
diff --git a/internal/ap/ap_test.go b/internal/ap/ap_test.go
index 583a37c53..0a9f66ca6 100644
--- a/internal/ap/ap_test.go
+++ b/internal/ap/ap_test.go
@@ -18,8 +18,10 @@
package ap_test
import (
+ "bytes"
"context"
"encoding/json"
+ "io"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/pub"
@@ -187,7 +189,10 @@ func (suite *APTestSuite) noteWithHashtags1() ap.Statusable {
}
}`)
- statusable, err := ap.ResolveStatusable(context.Background(), noteJson)
+ statusable, err := ap.ResolveStatusable(
+ context.Background(),
+ io.NopCloser(bytes.NewReader(noteJson)),
+ )
if err != nil {
suite.FailNow(err.Error())
}
diff --git a/internal/ap/collections.go b/internal/ap/collections.go
index ba3887a5b..da789e179 100644
--- a/internal/ap/collections.go
+++ b/internal/ap/collections.go
@@ -26,6 +26,24 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/paging"
)
+// TODO: replace must of this logic with just
+// using extractIRIs() on the iterator types.
+
+// ToCollectionIterator attempts to resolve the given vocab type as a Collection
+// like object and wrap in a standardised interface in order to iterate its contents.
+func ToCollectionIterator(t vocab.Type) (CollectionIterator, error) {
+ switch name := t.GetTypeName(); name {
+ case ObjectCollection:
+ t := t.(vocab.ActivityStreamsCollection)
+ return WrapCollection(t), nil
+ case ObjectOrderedCollection:
+ t := t.(vocab.ActivityStreamsOrderedCollection)
+ return WrapOrderedCollection(t), nil
+ default:
+ return nil, fmt.Errorf("%T(%s) was not Collection-like", t, name)
+ }
+}
+
// ToCollectionPageIterator attempts to resolve the given vocab type as a CollectionPage
// like object and wrap in a standardised interface in order to iterate its contents.
func ToCollectionPageIterator(t vocab.Type) (CollectionPageIterator, error) {
@@ -41,6 +59,16 @@ func ToCollectionPageIterator(t vocab.Type) (CollectionPageIterator, error) {
}
}
+// WrapCollection wraps an ActivityStreamsCollection in a standardised collection interface.
+func WrapCollection(collection vocab.ActivityStreamsCollection) CollectionIterator {
+ return ®ularCollectionIterator{ActivityStreamsCollection: collection}
+}
+
+// WrapOrderedCollection wraps an ActivityStreamsOrderedCollection in a standardised collection interface.
+func WrapOrderedCollection(collection vocab.ActivityStreamsOrderedCollection) CollectionIterator {
+ return &orderedCollectionIterator{ActivityStreamsOrderedCollection: collection}
+}
+
// WrapCollectionPage wraps an ActivityStreamsCollectionPage in a standardised collection page interface.
func WrapCollectionPage(page vocab.ActivityStreamsCollectionPage) CollectionPageIterator {
return ®ularCollectionPageIterator{ActivityStreamsCollectionPage: page}
@@ -51,6 +79,90 @@ func WrapOrderedCollectionPage(page vocab.ActivityStreamsOrderedCollectionPage)
return &orderedCollectionPageIterator{ActivityStreamsOrderedCollectionPage: page}
}
+// regularCollectionIterator implements CollectionIterator
+// for the vocab.ActivitiyStreamsCollection type.
+type regularCollectionIterator struct {
+ vocab.ActivityStreamsCollection
+ items vocab.ActivityStreamsItemsPropertyIterator
+ once bool // only init items once
+}
+
+func (iter *regularCollectionIterator) NextItem() TypeOrIRI {
+ if !iter.initItems() {
+ return nil
+ }
+ cur := iter.items
+ iter.items = iter.items.Next()
+ return cur
+}
+
+func (iter *regularCollectionIterator) PrevItem() TypeOrIRI {
+ if !iter.initItems() {
+ return nil
+ }
+ cur := iter.items
+ iter.items = iter.items.Prev()
+ return cur
+}
+
+func (iter *regularCollectionIterator) initItems() bool {
+ if iter.once {
+ return (iter.items != nil)
+ }
+ iter.once = true
+ if iter.ActivityStreamsCollection == nil {
+ return false // no page set
+ }
+ items := iter.GetActivityStreamsItems()
+ if items == nil {
+ return false // no items found
+ }
+ iter.items = items.Begin()
+ return (iter.items != nil)
+}
+
+// orderedCollectionIterator implements CollectionIterator
+// for the vocab.ActivitiyStreamsOrderedCollection type.
+type orderedCollectionIterator struct {
+ vocab.ActivityStreamsOrderedCollection
+ items vocab.ActivityStreamsOrderedItemsPropertyIterator
+ once bool // only init items once
+}
+
+func (iter *orderedCollectionIterator) NextItem() TypeOrIRI {
+ if !iter.initItems() {
+ return nil
+ }
+ cur := iter.items
+ iter.items = iter.items.Next()
+ return cur
+}
+
+func (iter *orderedCollectionIterator) PrevItem() TypeOrIRI {
+ if !iter.initItems() {
+ return nil
+ }
+ cur := iter.items
+ iter.items = iter.items.Prev()
+ return cur
+}
+
+func (iter *orderedCollectionIterator) initItems() bool {
+ if iter.once {
+ return (iter.items != nil)
+ }
+ iter.once = true
+ if iter.ActivityStreamsOrderedCollection == nil {
+ return false // no page set
+ }
+ items := iter.GetActivityStreamsOrderedItems()
+ if items == nil {
+ return false // no items found
+ }
+ iter.items = items.Begin()
+ return (iter.items != nil)
+}
+
// regularCollectionPageIterator implements CollectionPageIterator
// for the vocab.ActivitiyStreamsCollectionPage type.
type regularCollectionPageIterator struct {
diff --git a/internal/ap/collections_test.go b/internal/ap/collections_test.go
new file mode 100644
index 000000000..87a5bb057
--- /dev/null
+++ b/internal/ap/collections_test.go
@@ -0,0 +1,148 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ap_test
+
+import (
+ "net/url"
+ "slices"
+ "testing"
+
+ "github.com/superseriousbusiness/activity/pub"
+ "github.com/superseriousbusiness/activity/streams"
+ "github.com/superseriousbusiness/activity/streams/vocab"
+ "github.com/superseriousbusiness/gotosocial/internal/ap"
+)
+
+var testIteratorIRIs = [][]string{
+ {
+ "https://google.com",
+ "https://mastodon.social",
+ "http://naughty.naughty.website/heres/the/porn",
+ "https://god.monarchies.suck?yes=they&really=do",
+ },
+ {
+ // zero length
+ },
+ {
+ "https://superseriousbusiness.org",
+ "http://gotosocial.tv/@slothsgonewild",
+ },
+}
+
+func TestToCollectionIterator(t *testing.T) {
+ for _, iris := range testIteratorIRIs {
+ testToCollectionIterator(t, toCollection(iris), "", iris)
+ testToCollectionIterator(t, toOrderedCollection(iris), "", iris)
+ }
+ testToCollectionIterator(t, streams.NewActivityStreamsAdd(), "*typeadd.ActivityStreamsAdd(Add) was not Collection-like", nil)
+ testToCollectionIterator(t, streams.NewActivityStreamsBlock(), "*typeblock.ActivityStreamsBlock(Block) was not Collection-like", nil)
+}
+
+func TestToCollectionPageIterator(t *testing.T) {
+ for _, iris := range testIteratorIRIs {
+ testToCollectionPageIterator(t, toCollectionPage(iris), "", iris)
+ testToCollectionPageIterator(t, toOrderedCollectionPage(iris), "", iris)
+ }
+ testToCollectionPageIterator(t, streams.NewActivityStreamsAdd(), "*typeadd.ActivityStreamsAdd(Add) was not CollectionPage-like", nil)
+ testToCollectionPageIterator(t, streams.NewActivityStreamsBlock(), "*typeblock.ActivityStreamsBlock(Block) was not CollectionPage-like", nil)
+}
+
+func testToCollectionIterator(t *testing.T, in vocab.Type, expectErr string, expectIRIs []string) {
+ collect, err := ap.ToCollectionIterator(in)
+ if !errCheck(err, expectErr) {
+ t.Fatalf("did not return expected error: expect=%v receive=%v", expectErr, err)
+ }
+ iris := gatherFromIterator(collect)
+ if !slices.Equal(iris, expectIRIs) {
+ t.Fatalf("did not return expected iris: expect=%v receive=%v", expectIRIs, iris)
+ }
+}
+
+func testToCollectionPageIterator(t *testing.T, in vocab.Type, expectErr string, expectIRIs []string) {
+ page, err := ap.ToCollectionPageIterator(in)
+ if !errCheck(err, expectErr) {
+ t.Fatalf("did not return expected error: expect=%v receive=%v", expectErr, err)
+ }
+ iris := gatherFromIterator(page)
+ if !slices.Equal(iris, expectIRIs) {
+ t.Fatalf("did not return expected iris: expect=%v receive=%v", expectIRIs, iris)
+ }
+}
+
+func toCollection(iris []string) vocab.ActivityStreamsCollection {
+ collect := streams.NewActivityStreamsCollection()
+ collect.SetActivityStreamsItems(toItems(iris))
+ return collect
+}
+
+func toOrderedCollection(iris []string) vocab.ActivityStreamsOrderedCollection {
+ collect := streams.NewActivityStreamsOrderedCollection()
+ collect.SetActivityStreamsOrderedItems(toOrderedItems(iris))
+ return collect
+}
+
+func toCollectionPage(iris []string) vocab.ActivityStreamsCollectionPage {
+ page := streams.NewActivityStreamsCollectionPage()
+ page.SetActivityStreamsItems(toItems(iris))
+ return page
+}
+
+func toOrderedCollectionPage(iris []string) vocab.ActivityStreamsOrderedCollectionPage {
+ page := streams.NewActivityStreamsOrderedCollectionPage()
+ page.SetActivityStreamsOrderedItems(toOrderedItems(iris))
+ return page
+}
+
+func toItems(iris []string) vocab.ActivityStreamsItemsProperty {
+ items := streams.NewActivityStreamsItemsProperty()
+ for _, iri := range iris {
+ u, _ := url.Parse(iri)
+ items.AppendIRI(u)
+ }
+ return items
+}
+
+func toOrderedItems(iris []string) vocab.ActivityStreamsOrderedItemsProperty {
+ items := streams.NewActivityStreamsOrderedItemsProperty()
+ for _, iri := range iris {
+ u, _ := url.Parse(iri)
+ items.AppendIRI(u)
+ }
+ return items
+}
+
+func gatherFromIterator(iter ap.CollectionIterator) []string {
+ var iris []string
+ if iter == nil {
+ return nil
+ }
+ for item := iter.NextItem(); item != nil; item = iter.NextItem() {
+ id, _ := pub.ToId(item)
+ if id != nil {
+ iris = append(iris, id.String())
+ }
+ }
+ return iris
+}
+
+func errCheck(err error, str string) bool {
+ if err == nil {
+ return str == ""
+ }
+ return err.Error() == str
+}
diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go
index 811e09125..fa8e8a338 100644
--- a/internal/ap/interfaces.go
+++ b/internal/ap/interfaces.go
@@ -300,6 +300,15 @@ type ReplyToable interface {
WithInReplyTo
}
+// CollectionIterator represents the minimum interface for interacting with a
+// wrapped Collection or OrderedCollection in order to access next / prev items.
+type CollectionIterator interface {
+ vocab.Type
+
+ NextItem() TypeOrIRI
+ PrevItem() TypeOrIRI
+}
+
// CollectionPageIterator represents the minimum interface for interacting with a wrapped
// CollectionPage or OrderedCollectionPage in order to access both next / prev pages and items.
type CollectionPageIterator interface {
diff --git a/internal/ap/resolve.go b/internal/ap/resolve.go
index 20a858900..b2e866b6f 100644
--- a/internal/ap/resolve.go
+++ b/internal/ap/resolve.go
@@ -22,8 +22,8 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
- "sync"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
@@ -31,61 +31,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
-// mapPool is a memory pool of maps for JSON decoding.
-var mapPool = sync.Pool{
- New: func() any {
- return make(map[string]any)
- },
-}
-
-// getMap acquires a map from memory pool.
-func getMap() map[string]any {
- m := mapPool.Get().(map[string]any) //nolint
- return m
-}
-
-// putMap clears and places map back in pool.
-func putMap(m map[string]any) {
- if len(m) > int(^uint8(0)) {
- // don't pool overly
- // large maps.
- return
- }
- for k := range m {
- delete(m, k)
- }
- mapPool.Put(m)
-}
-
-// bytesToType tries to parse the given bytes slice
-// as a JSON ActivityPub type, failing if the input
-// bytes are not parseable as JSON, or do not parse
-// to an ActivityPub that we can understand.
-//
-// The given map pointer will also be populated with
-// the parsed JSON, to allow further processing.
-func bytesToType(
- ctx context.Context,
- b []byte,
- raw map[string]any,
-) (vocab.Type, error) {
- // Unmarshal the raw JSON bytes into a "raw" map.
- // This will fail if the input is not parseable
- // as JSON; eg., a remote has returned HTML as a
- // fallback response to an ActivityPub JSON request.
- if err := json.Unmarshal(b, &raw); err != nil {
- return nil, gtserror.NewfAt(3, "error unmarshalling bytes into json: %w", err)
- }
-
- // Resolve an ActivityStreams type.
- t, err := streams.ToType(ctx, raw)
- if err != nil {
- return nil, gtserror.NewfAt(3, "error resolving json into ap vocab type: %w", err)
- }
-
- return t, nil
-}
-
// ResolveActivity is a util function for pulling a pub.Activity type out of an incoming request body,
// returning the resolved activity type, error and whether to accept activity (false = transient i.e. ignore).
func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.WithCode) {
@@ -93,25 +38,23 @@ func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.With
// destination.
raw := getMap()
- // Tidy up when done.
- defer r.Body.Close()
+ // Decode data as JSON into 'raw' map
+ // and get the resolved AS vocab.Type.
+ // (this handles close of request body).
+ t, err := decodeType(r.Context(), r.Body, raw)
- // Decode the JSON body stream into "raw" map.
- if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
- err := gtserror.Newf("error decoding json: %w", err)
- return nil, false, gtserror.NewErrorInternalError(err)
- }
-
- // Resolve "raw" JSON to vocab.Type.
- t, err := streams.ToType(r.Context(), raw)
if err != nil {
+ // NOTE: if the error here was due to the response body
+ // ending early, the connection will have broken so it
+ // doesn't matter if we try to return 400 or 500, the
+ // error is mainly for our logging. tl;dr there's not a
+ // huge need to differentiate between those error types.
+
if !streams.IsUnmatchedErr(err) {
err := gtserror.Newf("error matching json to type: %w", err)
return nil, false, gtserror.NewErrorInternalError(err)
}
- // Respond with bad request; we just couldn't
- // match the type to one that we know about.
const text = "body json not resolvable as ActivityStreams type"
return nil, false, gtserror.NewErrorBadRequest(errors.New(text), text)
}
@@ -142,18 +85,19 @@ func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.With
return activity, true, nil
}
-// ResolveStatusable tries to resolve the given bytes into an ActivityPub Statusable representation.
-// It will then perform normalization on the Statusable.
+// ResolveStatusable tries to resolve the response data as an ActivityPub
+// Statusable representation. It will then perform normalization on the Statusable.
//
// Works for: Article, Document, Image, Video, Note, Page, Event, Place, Profile, Question.
-func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) {
+func ResolveStatusable(ctx context.Context, body io.ReadCloser) (Statusable, error) {
// Get "raw" map
// destination.
raw := getMap()
- // Convert raw bytes to an AP type.
- // This will also populate the map.
- t, err := bytesToType(ctx, b, raw)
+ // Decode data as JSON into 'raw' map
+ // and get the resolved AS vocab.Type.
+ // (this handles close of given body).
+ t, err := decodeType(ctx, body, raw)
if err != nil {
return nil, gtserror.SetWrongType(err)
}
@@ -183,18 +127,19 @@ func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) {
return statusable, nil
}
-// ResolveStatusable tries to resolve the given bytes into an ActivityPub Accountable representation.
-// It will then perform normalization on the Accountable.
+// ResolveAccountable tries to resolve the given reader into an ActivityPub
+// Accountable representation. It will then perform normalization on the Accountable.
//
// Works for: Application, Group, Organization, Person, Service
-func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) {
+func ResolveAccountable(ctx context.Context, body io.ReadCloser) (Accountable, error) {
// Get "raw" map
// destination.
raw := getMap()
- // Convert raw bytes to an AP type.
- // This will also populate the map.
- t, err := bytesToType(ctx, b, raw)
+ // Decode data as JSON into 'raw' map
+ // and get the resolved AS vocab.Type.
+ // (this handles close of given body).
+ t, err := decodeType(ctx, body, raw)
if err != nil {
return nil, gtserror.SetWrongType(err)
}
@@ -213,3 +158,104 @@ func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) {
return accountable, nil
}
+
+// ResolveCollection tries to resolve the given reader into an ActivityPub Collection-like
+// representation, then wrapping as abstracted iterator. Works for: Collection, OrderedCollection.
+func ResolveCollection(ctx context.Context, body io.ReadCloser) (CollectionIterator, error) {
+ // Get "raw" map
+ // destination.
+ raw := getMap()
+
+ // Decode data as JSON into 'raw' map
+ // and get the resolved AS vocab.Type.
+ // (this handles close of given body).
+ t, err := decodeType(ctx, body, raw)
+ if err != nil {
+ return nil, gtserror.SetWrongType(err)
+ }
+
+ // Release.
+ putMap(raw)
+
+ // Cast as as Collection-like.
+ return ToCollectionIterator(t)
+}
+
+// ResolveCollectionPage tries to resolve the given reader into an ActivityPub CollectionPage-like
+// representation, then wrapping as abstracted iterator. Works for: CollectionPage, OrderedCollectionPage.
+func ResolveCollectionPage(ctx context.Context, body io.ReadCloser) (CollectionPageIterator, error) {
+ // Get "raw" map
+ // destination.
+ raw := getMap()
+
+ // Decode data as JSON into 'raw' map
+ // and get the resolved AS vocab.Type.
+ // (this handles close of given body).
+ t, err := decodeType(ctx, body, raw)
+ if err != nil {
+ return nil, gtserror.SetWrongType(err)
+ }
+
+ // Release.
+ putMap(raw)
+
+ // Cast as as CollectionPage-like.
+ return ToCollectionPageIterator(t)
+}
+
+// emptydest is an empty JSON decode
+// destination useful for "noop" decodes
+// to check underlying reader is empty.
+var emptydest = &struct{}{}
+
+// decodeType tries to read and parse the data
+// at provided io.ReadCloser as a JSON ActivityPub
+// type, failing if not parseable as JSON or not
+// resolveable as one of our known AS types.
+//
+// NOTE: this function handles closing
+// given body when it is finished with.
+//
+// The given map pointer will also be populated with
+// the 'raw' JSON data, for further processing.
+func decodeType(
+ ctx context.Context,
+ body io.ReadCloser,
+ raw map[string]any,
+) (vocab.Type, error) {
+
+ // Wrap body in JSON decoder.
+ //
+ // We do this instead of using json.Unmarshal()
+ // so we can take advantage of the decoder's streamed
+ // check of input data as valid JSON. This means that
+ // in the cases of garbage input, or even just fallback
+ // HTML responses that were incorrectly content-type'd,
+ // we can error-out as soon as possible.
+ dec := json.NewDecoder(body)
+
+ // Unmarshal JSON source data into "raw" map.
+ if err := dec.Decode(&raw); err != nil {
+ _ = body.Close() // ensure closed.
+ return nil, gtserror.NewfAt(3, "error decoding into json: %w", err)
+ }
+
+ // Perform a secondary decode just to ensure we drained the
+ // entirety of the data source. Error indicates either extra
+ // trailing garbage, or multiple JSON values (invalid data).
+ if err := dec.Decode(emptydest); err != io.EOF {
+ _ = body.Close() // ensure closed.
+ return nil, gtserror.NewfAt(3, "data remaining after json")
+ }
+
+ // Done with body.
+ _ = body.Close()
+
+ // Resolve an ActivityStreams type.
+ t, err := streams.ToType(ctx, raw)
+ if err != nil {
+ return nil, gtserror.NewfAt(3, "error resolving json into ap vocab type: %w", err)
+ }
+
+ return t, nil
+}
diff --git a/internal/ap/resolve_test.go b/internal/ap/resolve_test.go
index bc32af7e4..aaf90ab0a 100644
--- a/internal/ap/resolve_test.go
+++ b/internal/ap/resolve_test.go
@@ -18,7 +18,9 @@
package ap_test
import (
+ "bytes"
"context"
+ "io"
"testing"
"github.com/stretchr/testify/suite"
@@ -33,7 +35,9 @@ type ResolveTestSuite struct {
func (suite *ResolveTestSuite) TestResolveDocumentAsStatusable() {
b := []byte(suite.typeToJson(suite.document1))
- statusable, err := ap.ResolveStatusable(context.Background(), b)
+ statusable, err := ap.ResolveStatusable(
+ context.Background(), io.NopCloser(bytes.NewReader(b)),
+ )
suite.NoError(err)
suite.NotNil(statusable)
}
@@ -41,7 +45,9 @@ func (suite *ResolveTestSuite) TestResolveDocumentAsStatusable() {
func (suite *ResolveTestSuite) TestResolveDocumentAsAccountable() {
b := []byte(suite.typeToJson(suite.document1))
- accountable, err := ap.ResolveAccountable(context.Background(), b)
+ accountable, err := ap.ResolveAccountable(
+ context.Background(), io.NopCloser(bytes.NewReader(b)),
+ )
suite.True(gtserror.IsWrongType(err))
suite.EqualError(err, "ResolveAccountable: cannot resolve vocab type *typedocument.ActivityStreamsDocument as accountable")
suite.Nil(accountable)
@@ -51,9 +57,11 @@ func (suite *ResolveTestSuite) TestResolveHTMLAsAccountable() {
b := []byte(`
.`)
- accountable, err := ap.ResolveAccountable(context.Background(), b)
+ accountable, err := ap.ResolveAccountable(
+ context.Background(), io.NopCloser(bytes.NewReader(b)),
+ )
suite.True(gtserror.IsWrongType(err))
- suite.EqualError(err, "ResolveAccountable: error unmarshalling bytes into json: invalid character '<' looking for beginning of value")
+ suite.EqualError(err, "ResolveAccountable: error decoding into json: invalid character '<' looking for beginning of value")
suite.Nil(accountable)
}
@@ -64,7 +72,9 @@ func (suite *ResolveTestSuite) TestResolveNonAPJSONAsAccountable() {
"pee pee":"poo poo"
}`)
- accountable, err := ap.ResolveAccountable(context.Background(), b)
+ accountable, err := ap.ResolveAccountable(
+ context.Background(), io.NopCloser(bytes.NewReader(b)),
+ )
suite.True(gtserror.IsWrongType(err))
suite.EqualError(err, "ResolveAccountable: error resolving json into ap vocab type: activity stream did not match any known types")
suite.Nil(accountable)
diff --git a/internal/ap/util.go b/internal/ap/util.go
index c810b7985..967a1659d 100644
--- a/internal/ap/util.go
+++ b/internal/ap/util.go
@@ -19,10 +19,39 @@ package ap
import (
"net/url"
+ "sync"
"github.com/superseriousbusiness/activity/streams/vocab"
)
+const mapmax = 256
+
+// mapPool is a memory pool
+// of maps for JSON decoding.
+var mapPool sync.Pool
+
+// getMap acquires a map from memory pool.
+func getMap() map[string]any {
+ v := mapPool.Get()
+ if v == nil {
+ // preallocate map of max-size.
+ m := make(map[string]any, mapmax)
+ v = m
+ }
+ return v.(map[string]any) //nolint
+}
+
+// putMap clears and places map back in pool.
+func putMap(m map[string]any) {
+ if len(m) > mapmax {
+ // drop maps beyond
+ // our maximum size.
+ return
+ }
+ clear(m)
+ mapPool.Put(m)
+}
+
// _TypeOrIRI wraps a vocab.Type to implement TypeOrIRI.
type _TypeOrIRI struct {
vocab.Type
diff --git a/internal/federation/authenticate.go b/internal/federation/authenticate.go
index 59281fa65..596233b19 100644
--- a/internal/federation/authenticate.go
+++ b/internal/federation/authenticate.go
@@ -23,6 +23,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
"net/url"
"time"
@@ -414,10 +415,19 @@ func (f *Federator) callForPubKey(
// The actual http call to the remote server is
// made right here by the Dereference function.
- pubKeyBytes, err := trans.Dereference(ctx, pubKeyID)
+ rsp, err := trans.Dereference(ctx, pubKeyID)
+
if err == nil {
- // No problem.
- return pubKeyBytes, nil
+ // Read the response body data.
+ b, err := io.ReadAll(rsp.Body)
+ _ = rsp.Body.Close() // done
+
+ if err != nil {
+ err := gtserror.Newf("error reading pubkey: %w", err)
+ return nil, gtserror.NewErrorInternalError(err)
+ }
+
+ return b, nil
}
if gtserror.StatusCode(err) == http.StatusGone {
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
index c3ad6be5e..10d15bca6 100644
--- a/internal/federation/dereferencing/account.go
+++ b/internal/federation/dereferencing/account.go
@@ -19,14 +19,12 @@ package dereferencing
import (
"context"
- "encoding/json"
"errors"
"io"
"net/url"
"time"
- "github.com/superseriousbusiness/activity/streams"
- "github.com/superseriousbusiness/activity/streams/vocab"
+ "github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -499,16 +497,17 @@ func (d *Dereferencer) enrichAccount(
case err == nil && account.Domain != accDomain:
// After webfinger, we now have correct account domain from which we can do a final DB check.
- alreadyAcct, err := d.state.DB.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
+ alreadyAcc, err := d.state.DB.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
- return nil, nil, gtserror.Newf("db err looking for account again after webfinger: %w", err)
+ return nil, nil, gtserror.Newf("db error getting account after webfinger: %w", err)
}
- if alreadyAcct != nil {
- // We had this account stored under
- // the discovered accountDomain.
+ if alreadyAcc != nil {
+ // We had this account stored
+ // under discovered accountDomain.
+ //
// Proceed with this account.
- account = alreadyAcct
+ account = alreadyAcc
}
// Whether we had the account or not, we
@@ -537,8 +536,9 @@ func (d *Dereferencer) enrichAccount(
)
}
+ // Check URI scheme ahead of time for more useful errs.
if uri.Scheme != "http" && uri.Scheme != "https" {
- err = errors.New("account URI scheme must be http or https")
+ err := errors.New("account URI scheme must be http or https")
return nil, nil, gtserror.Newf(
"invalid uri %q: %w",
account.URI, gtserror.SetUnretrievable(err),
@@ -567,20 +567,52 @@ func (d *Dereferencer) enrichAccount(
// We were not given any (partial) ActivityPub
// version of this account as a parameter.
// Dereference latest version of the account.
- b, err := tsport.Dereference(ctx, uri)
+ rsp, err := tsport.Dereference(ctx, uri)
if err != nil {
err := gtserror.Newf("error dereferencing %s: %w", uri, err)
return nil, nil, gtserror.SetUnretrievable(err)
}
- // Attempt to resolve ActivityPub acc from data.
- apubAcc, err = ap.ResolveAccountable(ctx, b)
+ // Attempt to resolve ActivityPub acc from response.
+ apubAcc, err = ap.ResolveAccountable(ctx, rsp.Body)
+
+ // Tidy up now done.
+ _ = rsp.Body.Close()
+
if err != nil {
- // ResolveAccountable will set Unretrievable/WrongType
+ // ResolveAccountable will set gtserror.WrongType
// on the returned error, so we don't need to do it here.
- err = gtserror.Newf("error resolving accountable from data for account %s: %w", uri, err)
+ err = gtserror.Newf("error resolving accountable %s: %w", uri, err)
return nil, nil, err
}
+
+ // Check whether input URI and final returned URI
+ // have changed (i.e. we followed some redirects).
+ if finalURIStr := rsp.Request.URL.String(); //
+ finalURIStr != uri.String() {
+
+ // NOTE: this URI check + database call is performed
+ // AFTER reading and closing response body, for performance.
+ //
+ // Check whether we have this account stored under *final* URI.
+ alreadyAcc, err := d.state.DB.GetAccountByURI(ctx, finalURIStr)
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
+ return nil, nil, gtserror.Newf("db error getting account after redirects: %w", err)
+ }
+
+ if alreadyAcc != nil {
+ // We had this account stored
+ // under discovered final URI.
+ //
+ // Proceed with this account.
+ account = alreadyAcc
+ }
+
+ // Update the input URI to
+ // the final determined URI
+ // for later URI checks.
+ uri = rsp.Request.URL
+ }
}
/*
@@ -1014,76 +1046,33 @@ func (d *Dereferencer) dereferenceAccountFeatured(ctx context.Context, requestUs
return err
}
- // Pre-fetch a transport for requesting username, used by later deref procedures.
- tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
- if err != nil {
- return gtserror.Newf("couldn't create transport: %w", err)
- }
-
- b, err := tsport.Dereference(ctx, uri)
+ collect, err := d.dereferenceCollection(ctx, requestUser, uri)
if err != nil {
return err
}
- m := make(map[string]interface{})
- if err := json.Unmarshal(b, &m); err != nil {
- return gtserror.Newf("error unmarshalling bytes into json: %w", err)
- }
-
- t, err := streams.ToType(ctx, m)
- if err != nil {
- return gtserror.Newf("error resolving json into ap vocab type: %w", err)
- }
-
- if t.GetTypeName() != ap.ObjectOrderedCollection {
- return gtserror.Newf("%s was not an OrderedCollection", uri)
- }
-
- collection, ok := t.(vocab.ActivityStreamsOrderedCollection)
- if !ok {
- return gtserror.New("couldn't coerce OrderedCollection")
- }
-
- items := collection.GetActivityStreamsOrderedItems()
- if items == nil {
- return gtserror.New("nil orderedItems")
- }
-
// Get previous pinned statuses (we'll need these later).
wasPinned, err := d.state.DB.GetAccountPinnedStatuses(ctx, account.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("error getting account pinned statuses: %w", err)
}
- statusURIs := make([]*url.URL, 0, items.Len())
- for iter := items.Begin(); iter != items.End(); iter = iter.Next() {
- var statusURI *url.URL
+ var statusURIs []*url.URL
- switch {
- case iter.IsActivityStreamsNote():
- // We got a whole Note. Extract the URI.
- if note := iter.GetActivityStreamsNote(); note != nil {
- if id := note.GetJSONLDId(); id != nil {
- statusURI = id.GetIRI()
- }
- }
- case iter.IsActivityStreamsArticle():
- // We got a whole Article. Extract the URI.
- if article := iter.GetActivityStreamsArticle(); article != nil {
- if id := article.GetJSONLDId(); id != nil {
- statusURI = id.GetIRI()
- }
- }
- default:
- // Try to get just the URI.
- statusURI = iter.GetIRI()
+ for {
+ // Get next collect item.
+ item := collect.NextItem()
+ if item == nil {
+ break
}
- if statusURI == nil {
+ // Check for available IRI.
+ itemIRI, _ := pub.ToId(item)
+ if itemIRI == nil {
continue
}
- if statusURI.Host != uri.Host {
+ if itemIRI.Host != uri.Host {
// If this status doesn't share a host with its featured
// collection URI, we shouldn't trust it. Just move on.
continue
@@ -1093,13 +1082,13 @@ func (d *Dereferencer) dereferenceAccountFeatured(ctx context.Context, requestUs
// We do this here so that even if we can't get
// the status in the next part for some reason,
// we still know it was *meant* to be pinned.
- statusURIs = append(statusURIs, statusURI)
+ statusURIs = append(statusURIs, itemIRI)
// Search for status by URI. Note this may return an existing model
// we have stored with an error from attempted update, so check both.
- status, _, _, err := d.getStatusByURI(ctx, requestUser, statusURI)
+ status, _, _, err := d.getStatusByURI(ctx, requestUser, itemIRI)
if err != nil {
- log.Errorf(ctx, "error getting status from featured collection %s: %v", statusURI, err)
+ log.Errorf(ctx, "error getting status from featured collection %s: %v", itemIRI, err)
if status == nil {
// This is only unactionable
@@ -1108,20 +1097,23 @@ func (d *Dereferencer) dereferenceAccountFeatured(ctx context.Context, requestUs
}
}
- // If the status was already pinned, we don't need to do anything.
+ // If the status was already pinned,
+ // we don't need to do anything.
if !status.PinnedAt.IsZero() {
continue
}
- if status.AccountID != account.ID {
+ if status.AccountURI != account.URI {
// Someone's pinned a status that doesn't
// belong to them, this doesn't work for us.
continue
}
if status.BoostOfID != "" {
- // Someone's pinned a boost. This also
- // doesn't work for us.
+ // Someone's pinned a boost. This
+ // also doesn't work for us. (note
+ // we check using BoostOfID since
+ // BoostOfURI isn't *always* set).
continue
}
diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go
index 8d082105b..02b1d5e5c 100644
--- a/internal/federation/dereferencing/announce.go
+++ b/internal/federation/dereferencing/announce.go
@@ -83,6 +83,10 @@ func (d *Dereferencer) EnrichAnnounce(
return nil, gtserror.Newf("error generating id: %w", err)
}
+ // Set boost_of_uri again in case the
+ // original URI was an indirect link.
+ boost.BoostOfURI = target.URI
+
// Populate remaining fields on
// the boost wrapper using target.
boost.Content = target.Content
@@ -101,10 +105,10 @@ func (d *Dereferencer) EnrichAnnounce(
boost.Replyable = target.Replyable
boost.Likeable = target.Likeable
- // Store the boost wrapper status.
+ // Store the boost wrapper status in database.
switch err = d.state.DB.PutStatus(ctx, boost); {
case err == nil:
- // All good baby.
+ // all groovy.
case errors.Is(err, db.ErrAlreadyExists):
uri := boost.URI
@@ -119,8 +123,7 @@ func (d *Dereferencer) EnrichAnnounce(
)
}
- default:
- // Proper database error.
+ default: // Proper database error.
return nil, gtserror.Newf("db error inserting status: %w", err)
}
diff --git a/internal/federation/dereferencing/collectionpage.go b/internal/federation/dereferencing/collection.go
similarity index 77%
rename from internal/federation/dereferencing/collectionpage.go
rename to internal/federation/dereferencing/collection.go
index dc5c68273..07f56c952 100644
--- a/internal/federation/dereferencing/collectionpage.go
+++ b/internal/federation/dereferencing/collection.go
@@ -19,16 +19,42 @@ package dereferencing
import (
"context"
- "encoding/json"
"net/url"
- "github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
+// dereferenceCollectionPage returns the activitystreams Collection at the specified IRI, or an error if something goes wrong.
+func (d *Dereferencer) dereferenceCollection(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionIterator, error) {
+ if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
+ return nil, gtserror.Newf("domain %s is blocked", pageIRI.Host)
+ }
+
+ transport, err := d.transportController.NewTransportForUsername(ctx, username)
+ if err != nil {
+ return nil, gtserror.Newf("error creating transport: %w", err)
+ }
+
+ rsp, err := transport.Dereference(ctx, pageIRI)
+ if err != nil {
+ return nil, gtserror.Newf("error deferencing %s: %w", pageIRI.String(), err)
+ }
+
+ collect, err := ap.ResolveCollection(ctx, rsp.Body)
+
+ // Tidy up rsp body.
+ _ = rsp.Body.Close()
+
+ if err != nil {
+ return nil, gtserror.Newf("error resolving collection %s: %w", pageIRI.String(), err)
+ }
+
+ return collect, nil
+}
+
// dereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
func (d *Dereferencer) dereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageIterator, error) {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
@@ -40,24 +66,18 @@ func (d *Dereferencer) dereferenceCollectionPage(ctx context.Context, username s
return nil, gtserror.Newf("error creating transport: %w", err)
}
- b, err := transport.Dereference(ctx, pageIRI)
+ rsp, err := transport.Dereference(ctx, pageIRI)
if err != nil {
return nil, gtserror.Newf("error deferencing %s: %w", pageIRI.String(), err)
}
- m := make(map[string]interface{})
- if err := json.Unmarshal(b, &m); err != nil {
- return nil, gtserror.Newf("error unmarshalling bytes into json: %w", err)
- }
+ page, err := ap.ResolveCollectionPage(ctx, rsp.Body)
- t, err := streams.ToType(ctx, m)
- if err != nil {
- return nil, gtserror.Newf("error resolving json into ap vocab type: %w", err)
- }
+ // Tidy up rsp body.
+ _ = rsp.Body.Close()
- page, err := ap.ToCollectionPageIterator(t)
if err != nil {
- return nil, gtserror.Newf("error resolving vocab type as page: %w", err)
+ return nil, gtserror.Newf("error resolving collection page %s: %w", pageIRI.String(), err)
}
return page, nil
diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
index e3f97553d..397d2aa28 100644
--- a/internal/federation/dereferencing/status.go
+++ b/internal/federation/dereferencing/status.go
@@ -393,16 +393,51 @@ func (d *Dereferencer) enrichStatus(
if apubStatus == nil {
// Dereference latest version of the status.
- b, err := tsport.Dereference(ctx, uri)
+ rsp, err := tsport.Dereference(ctx, uri)
if err != nil {
err := gtserror.Newf("error dereferencing %s: %w", uri, err)
return nil, nil, gtserror.SetUnretrievable(err)
}
- // Attempt to resolve ActivityPub status from data.
- apubStatus, err = ap.ResolveStatusable(ctx, b)
+ // Attempt to resolve ActivityPub status from response.
+ apubStatus, err = ap.ResolveStatusable(ctx, rsp.Body)
+
+ // Tidy up now done.
+ _ = rsp.Body.Close()
+
if err != nil {
- return nil, nil, gtserror.Newf("error resolving statusable from data for account %s: %w", uri, err)
+ // ResolveStatusable will set gtserror.WrongType
+ // on the returned error, so we don't need to do it here.
+ err = gtserror.Newf("error resolving statusable %s: %w", uri, err)
+ return nil, nil, err
+ }
+
+ // Check whether input URI and final returned URI
+ // have changed (i.e. we followed some redirects).
+ if finalURIStr := rsp.Request.URL.String(); //
+ finalURIStr != uri.String() {
+
+ // NOTE: this URI check + database call is performed
+ // AFTER reading and closing response body, for performance.
+ //
+ // Check whether we have this status stored under *final* URI.
+ alreadyStatus, err := d.state.DB.GetStatusByURI(ctx, finalURIStr)
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
+ return nil, nil, gtserror.Newf("db error getting status after redirects: %w", err)
+ }
+
+ if alreadyStatus != nil {
+ // We had this status stored
+ // under discovered final URI.
+ //
+ // Proceed with this status.
+ status = alreadyStatus
+ }
+
+ // Update the input URI to
+ // the final determined URI
+ // for later URI checks.
+ uri = rsp.Request.URL
}
}
diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go
index 28f7ffa8a..e528581c9 100644
--- a/internal/federation/dereferencing/thread.go
+++ b/internal/federation/dereferencing/thread.go
@@ -33,7 +33,7 @@ import (
// maxIter defines how many iterations of descendants or
// ancesters we are willing to follow before returning error.
-const maxIter = 1000
+const maxIter = 512
// dereferenceThread handles dereferencing status thread after
// fetch. Passing off appropriate parts to be enqueued for async
@@ -98,16 +98,10 @@ func (d *Dereferencer) DereferenceStatusAncestors(ctx context.Context, username
return nil
}
+ // Apparent current parent URI to log fields.
l = l.WithField("parent", current.InReplyToURI)
l.Trace("following status ancestor")
- // Parse status parent URI for later use.
- uri, err := url.Parse(current.InReplyToURI)
- if err != nil {
- l.Warnf("invalid uri: %v", err)
- return nil
- }
-
// Check whether this parent has already been deref'd.
if _, ok := derefdStatuses[current.InReplyToURI]; ok {
l.Warn("self referencing status ancestor")
@@ -117,6 +111,13 @@ func (d *Dereferencer) DereferenceStatusAncestors(ctx context.Context, username
// Add this status's parent URI to map of deref'd.
derefdStatuses[current.InReplyToURI] = struct{}{}
+ // Parse status parent URI for later use.
+ uri, err := url.Parse(current.InReplyToURI)
+ if err != nil {
+ l.Warnf("invalid uri: %v", err)
+ return nil
+ }
+
// Fetch parent status by current's reply URI, this handles
// case of existing (updating if necessary) or a new status.
parent, _, _, err := d.getStatusByURI(ctx, username, uri)
@@ -129,6 +130,7 @@ func (d *Dereferencer) DereferenceStatusAncestors(ctx context.Context, username
// view the status (it's followers-only and
// we don't follow, for example).
case code == http.StatusNotFound:
+
// If this reply is followers-only or stricter,
// we can safely assume the status it replies
// to is also followers only or stricter.
@@ -153,31 +155,43 @@ func (d *Dereferencer) DereferenceStatusAncestors(ctx context.Context, username
// the now-gone parent.
case code == http.StatusGone:
l.Trace("status orphaned")
- current.InReplyToID = ""
- current.InReplyToURI = ""
- current.InReplyToAccountID = ""
current.InReplyTo = nil
current.InReplyToAccount = nil
- if err := d.state.DB.UpdateStatus(ctx,
+ return d.updateStatusParent(ctx,
current,
- "in_reply_to_id",
- "in_reply_to_uri",
- "in_reply_to_account_id",
- ); err != nil {
- return gtserror.Newf("db error updating status %s: %w", current.ID, err)
- }
- return nil
+ "", // status ID
+ "", // status URI
+ "", // account ID
+ )
// An error was returned for a status during
// an attempted NEW dereference, return here.
- case err != nil && current.InReplyToID == "":
+ //
+ // NOTE: this will catch all cases of a nil
+ // parent, all cases below can safely assume
+ // a non-nil parent in their code logic.
+ case err != nil && parent == nil:
return gtserror.Newf("error dereferencing new %s: %w", current.InReplyToURI, err)
// An error was returned for an existing parent,
// we simply treat this as a temporary situation.
- // (we fallback to using existing parent status).
case err != nil:
l.Errorf("error getting parent: %v", err)
+ }
+
+ // Start a new switch case
+ // as the following scenarios
+ // are possible with / without
+ // any returned error.
+ switch {
+
+ // The current status is using an indirect URL
+ // in order to reference the parent. This is just
+ // weird and broken... Leave the URI in place but
+ // don't link the statuses via database IDs as it
+ // could cause all sorts of unexpected situations.
+ case current.InReplyToURI != parent.URI:
+ l.Errorf("indirect in_reply_to_uri => %s", parent.URI)
// The ID has changed for currently stored parent ID
// (which may be empty, if new!) and fetched version.
@@ -185,17 +199,14 @@ func (d *Dereferencer) DereferenceStatusAncestors(ctx context.Context, username
// Update the current's inReplyTo fields to parent.
case current.InReplyToID != parent.ID:
l.Tracef("parent changed %s => %s", current.InReplyToID, parent.ID)
- current.InReplyToAccountID = parent.AccountID
current.InReplyToAccount = parent.Account
- current.InReplyToURI = parent.URI
- current.InReplyToID = parent.ID
- if err := d.state.DB.UpdateStatus(ctx,
+ if err := d.updateStatusParent(ctx,
current,
- "in_reply_to_id",
- "in_reply_to_uri",
- "in_reply_to_account_id",
+ parent.ID,
+ parent.URI,
+ parent.AccountID,
); err != nil {
- return gtserror.Newf("db error updating status %s: %w", current.ID, err)
+ return err
}
}
@@ -384,3 +395,26 @@ stackLoop:
return gtserror.Newf("reached %d descendant iterations for %q", maxIter, statusIRIStr)
}
+
+// updateStatusParent updates the given status' parent
+// status URI, ID and account ID to given values in DB.
+func (d *Dereferencer) updateStatusParent(
+ ctx context.Context,
+ status *gtsmodel.Status,
+ parentStatusID string,
+ parentStatusURI string,
+ parentAccountID string,
+) error {
+ status.InReplyToAccountID = parentAccountID
+ status.InReplyToURI = parentStatusURI
+ status.InReplyToID = parentStatusID
+ if err := d.state.DB.UpdateStatus(ctx,
+ status,
+ "in_reply_to_id",
+ "in_reply_to_uri",
+ "in_reply_to_account_id",
+ ); err != nil {
+ return gtserror.Newf("error updating status %s: %w", status.URI, err)
+ }
+ return nil
+}
diff --git a/internal/transport/controller.go b/internal/transport/controller.go
index 32ef6d7c2..81022596a 100644
--- a/internal/transport/controller.go
+++ b/internal/transport/controller.go
@@ -18,11 +18,15 @@
package transport
import (
+ "bytes"
"context"
"crypto/rsa"
"crypto/x509"
"encoding/json"
+ "errors"
"fmt"
+ "io"
+ "net/http"
"net/url"
"runtime"
@@ -31,6 +35,7 @@ import (
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/httpclient"
"github.com/superseriousbusiness/gotosocial/internal/state"
@@ -150,36 +155,73 @@ func (c *controller) NewTransportForUsername(ctx context.Context, username strin
// account on this instance, without making any external api/http calls.
//
// It is passed to new transports, and should only be invoked when the iri.Host == this host.
-func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) (*http.Response, error) {
followers, err := c.fedDB.Followers(ctx, iri)
- if err != nil {
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, err
}
+ if followers == nil {
+ // Return a generic 404 not found response.
+ rsp := craftResponse(iri, http.StatusNotFound)
+ return rsp, nil
+ }
+
i, err := ap.Serialize(followers)
if err != nil {
return nil, err
}
- return json.Marshal(i)
+ b, err := json.Marshal(i)
+ if err != nil {
+ return nil, err
+ }
+
+ // Return a response with AS data as body.
+ rsp := craftResponse(iri, http.StatusOK)
+ rsp.Body = io.NopCloser(bytes.NewReader(b))
+ return rsp, nil
}
// dereferenceLocalUser is a shortcut to dereference followers an account on
// this instance, without making any external api/http calls.
//
// It is passed to new transports, and should only be invoked when the iri.Host == this host.
-func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) (*http.Response, error) {
user, err := c.fedDB.Get(ctx, iri)
- if err != nil {
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, err
}
+ if user == nil {
+ // Return a generic 404 not found response.
+ rsp := craftResponse(iri, http.StatusNotFound)
+ return rsp, nil
+ }
+
i, err := ap.Serialize(user)
if err != nil {
return nil, err
}
- return json.Marshal(i)
+ b, err := json.Marshal(i)
+ if err != nil {
+ return nil, err
+ }
+
+ // Return a response with AS data as body.
+ rsp := craftResponse(iri, http.StatusOK)
+ rsp.Body = io.NopCloser(bytes.NewReader(b))
+ return rsp, nil
+}
+
+func craftResponse(url *url.URL, code int) *http.Response {
+ rsp := new(http.Response)
+ rsp.Request = new(http.Request)
+ rsp.Request.URL = url
+ rsp.Status = http.StatusText(code)
+ rsp.StatusCode = code
+ return rsp
}
// privkeyToPublicStr will create a string representation of RSA public key from private.
diff --git a/internal/transport/dereference.go b/internal/transport/dereference.go
index 3a33a81ad..efd3f0fbf 100644
--- a/internal/transport/dereference.go
+++ b/internal/transport/dereference.go
@@ -19,7 +19,6 @@ package transport
import (
"context"
- "io"
"net/http"
"net/url"
@@ -29,7 +28,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
-func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (t *transport) Dereference(ctx context.Context, iri *url.URL) (*http.Response, error) {
// if the request is to us, we can shortcut for certain URIs rather than going through
// the normal request flow, thereby saving time and energy
if iri.Host == config.GetHost() {
@@ -62,18 +61,20 @@ func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, erro
if err != nil {
return nil, err
}
- defer rsp.Body.Close()
// Ensure a non-error status response.
if rsp.StatusCode != http.StatusOK {
- return nil, gtserror.NewFromResponse(rsp)
+ err := gtserror.NewFromResponse(rsp)
+ _ = rsp.Body.Close() // done with body
+ return nil, err
}
// Ensure that the incoming request content-type is expected.
if ct := rsp.Header.Get("Content-Type"); !apiutil.ASContentType(ct) {
err := gtserror.Newf("non activity streams response: %s", ct)
+ _ = rsp.Body.Close() // done with body
return nil, gtserror.SetMalformed(err)
}
- return io.ReadAll(rsp.Body)
+ return rsp, nil
}
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index a2a9dc23c..99972fe25 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -65,7 +65,7 @@ type Transport interface {
GET(*http.Request) (*http.Response, error)
// Dereference fetches the ActivityStreams object located at this IRI with a GET request.
- Dereference(ctx context.Context, iri *url.URL) ([]byte, error)
+ Dereference(ctx context.Context, iri *url.URL) (*http.Response, error)
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error)
diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go
index fc8cd19a0..2ee2f9607 100644
--- a/internal/typeutils/astointernal_test.go
+++ b/internal/typeutils/astointernal_test.go
@@ -18,9 +18,11 @@
package typeutils_test
import (
+ "bytes"
"context"
"encoding/json"
"fmt"
+ "io"
"testing"
"github.com/stretchr/testify/suite"
@@ -39,11 +41,11 @@ func (suite *ASToInternalTestSuite) jsonToType(in string) vocab.Type {
ctx := context.Background()
b := []byte(in)
- if accountable, err := ap.ResolveAccountable(ctx, b); err == nil {
+ if accountable, err := ap.ResolveAccountable(ctx, io.NopCloser(bytes.NewReader(b))); err == nil {
return accountable
}
- if statusable, err := ap.ResolveStatusable(ctx, b); err == nil {
+ if statusable, err := ap.ResolveStatusable(ctx, io.NopCloser(bytes.NewReader(b))); err == nil {
return statusable
}
diff --git a/vendor/github.com/superseriousbusiness/activity/pub/base_actor.go b/vendor/github.com/superseriousbusiness/activity/pub/base_actor.go
index a89b05cfb..dc79c3b6e 100644
--- a/vendor/github.com/superseriousbusiness/activity/pub/base_actor.go
+++ b/vendor/github.com/superseriousbusiness/activity/pub/base_actor.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io/ioutil"
"net/http"
"net/url"
@@ -68,12 +67,7 @@ func NewSocialActor(c CommonBehavior,
clock Clock) Actor {
return &baseActor{
// Use SideEffectActor without s2s.
- delegate: &SideEffectActor{
- common: c,
- c2s: c2s,
- db: db,
- clock: clock,
- },
+ delegate: NewSideEffectActor(c, nil, c2s, db, clock),
enableSocialProtocol: true,
clock: clock,
}
@@ -98,12 +92,7 @@ func NewFederatingActor(c CommonBehavior,
return &baseActorFederating{
baseActor{
// Use SideEffectActor without c2s.
- delegate: &SideEffectActor{
- common: c,
- s2s: s2s,
- db: db,
- clock: clock,
- },
+ delegate: NewSideEffectActor(c, s2s, nil, db, clock),
enableFederatedProtocol: true,
clock: clock,
},
@@ -195,14 +184,10 @@ func (b *baseActor) PostInboxScheme(c context.Context, w http.ResponseWriter, r
// Begin processing the request, but have not yet applied
// authorization (ex: blocks). Obtain the activity reject unknown
// activities.
- raw, err := ioutil.ReadAll(r.Body)
+ m, err := readActivityPubReq(r)
if err != nil {
return true, err
}
- var m map[string]interface{}
- if err = json.Unmarshal(raw, &m); err != nil {
- return true, err
- }
asValue, err := streams.ToType(c, m)
if err != nil && !streams.IsUnmatchedErr(err) {
return true, err
@@ -340,14 +325,10 @@ func (b *baseActor) PostOutboxScheme(c context.Context, w http.ResponseWriter, r
return true, nil
}
// Everything is good to begin processing the request.
- raw, err := ioutil.ReadAll(r.Body)
+ m, err := readActivityPubReq(r)
if err != nil {
return true, err
}
- var m map[string]interface{}
- if err = json.Unmarshal(raw, &m); err != nil {
- return true, err
- }
// Note that converting to a Type will NOT successfully convert types
// not known to go-fed. This prevents accidentally wrapping an Activity
// type unknown to go-fed in a Create below. Instead,
diff --git a/vendor/github.com/superseriousbusiness/activity/pub/federating_wrapped_callbacks.go b/vendor/github.com/superseriousbusiness/activity/pub/federating_wrapped_callbacks.go
index fffee8b81..fe1bf3c8e 100644
--- a/vendor/github.com/superseriousbusiness/activity/pub/federating_wrapped_callbacks.go
+++ b/vendor/github.com/superseriousbusiness/activity/pub/federating_wrapped_callbacks.go
@@ -2,7 +2,6 @@ package pub
import (
"context"
- "encoding/json"
"fmt"
"net/url"
@@ -244,12 +243,12 @@ func (w FederatingWrappedCallbacks) create(c context.Context, a vocab.ActivitySt
if err != nil {
return err
}
- b, err := tport.Dereference(c, iter.GetIRI())
+ resp, err := tport.Dereference(c, iter.GetIRI())
if err != nil {
return err
}
- var m map[string]interface{}
- if err = json.Unmarshal(b, &m); err != nil {
+ m, err := readActivityPubResp(resp)
+ if err != nil {
return err
}
t, err = streams.ToType(c, m)
@@ -514,12 +513,12 @@ func (w FederatingWrappedCallbacks) accept(c context.Context, a vocab.ActivitySt
if err != nil {
return err
}
- b, err := tport.Dereference(c, iter.GetIRI())
+ resp, err := tport.Dereference(c, iter.GetIRI())
if err != nil {
return err
}
- var m map[string]interface{}
- if err = json.Unmarshal(b, &m); err != nil {
+ m, err := readActivityPubResp(resp)
+ if err != nil {
return err
}
t, err = streams.ToType(c, m)
diff --git a/vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go b/vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go
index 4fe300832..4062cf507 100644
--- a/vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go
+++ b/vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go
@@ -629,14 +629,14 @@ func (a *SideEffectActor) hasInboxForwardingValues(c context.Context, inboxIRI *
if err != nil {
return false, err
}
- b, err := tport.Dereference(c, iri)
+ resp, err := tport.Dereference(c, iri)
if err != nil {
// Do not fail the entire process if the data is
// missing.
continue
}
- var m map[string]interface{}
- if err = json.Unmarshal(b, &m); err != nil {
+ m, err := readActivityPubResp(resp)
+ if err != nil {
return false, err
}
t, err := streams.ToType(c, m)
@@ -855,13 +855,14 @@ func (a *SideEffectActor) resolveActors(c context.Context, t Transport, r []*url
// The returned actor could be nil, if it wasn't an actor (ex: a Collection or
// OrderedCollection).
func (a *SideEffectActor) dereferenceForResolvingInboxes(c context.Context, t Transport, actorIRI *url.URL) (actor vocab.Type, moreActorIRIs []*url.URL, err error) {
- var resp []byte
+ var resp *http.Response
resp, err = t.Dereference(c, actorIRI)
if err != nil {
return
}
var m map[string]interface{}
- if err = json.Unmarshal(resp, &m); err != nil {
+ m, err = readActivityPubResp(resp)
+ if err != nil {
return
}
actor, err = streams.ToType(c, m)
diff --git a/vendor/github.com/superseriousbusiness/activity/pub/transport.go b/vendor/github.com/superseriousbusiness/activity/pub/transport.go
index bdc58a97a..a770b8b46 100644
--- a/vendor/github.com/superseriousbusiness/activity/pub/transport.go
+++ b/vendor/github.com/superseriousbusiness/activity/pub/transport.go
@@ -5,7 +5,6 @@ import (
"context"
"crypto"
"fmt"
- "io/ioutil"
"net/http"
"net/url"
"strings"
@@ -40,11 +39,13 @@ func isSuccess(code int) bool {
//
// It may be reused multiple times, but never concurrently.
type Transport interface {
- // Dereference fetches the ActivityStreams object located at this IRI
- // with a GET request.
- Dereference(c context.Context, iri *url.URL) ([]byte, error)
+ // Dereference fetches the ActivityStreams object located at this IRI with
+ // a GET request. Note that Response will only be returned on status = OK.
+ Dereference(c context.Context, iri *url.URL) (*http.Response, error)
+
// Deliver sends an ActivityStreams object.
Deliver(c context.Context, b []byte, to *url.URL) error
+
// BatchDeliver sends an ActivityStreams object to multiple recipients.
BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error
}
@@ -107,9 +108,8 @@ func NewHttpSigTransport(
}
}
-// Dereference sends a GET request signed with an HTTP Signature to obtain an
-// ActivityStreams value.
-func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) ([]byte, error) {
+// Dereference sends a GET request signed with an HTTP Signature to obtain an ActivityStreams value.
+func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) (*http.Response, error) {
req, err := http.NewRequest("GET", iri.String(), nil)
if err != nil {
return nil, err
@@ -130,11 +130,11 @@ func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) ([]byte,
if err != nil {
return nil, err
}
- defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
+ _ = resp.Body.Close()
return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
}
- return ioutil.ReadAll(resp.Body)
+ return resp, nil
}
// Deliver sends a POST request with an HTTP Signature.
@@ -166,8 +166,7 @@ func (h HttpSigTransport) Deliver(c context.Context, b []byte, to *url.URL) erro
return nil
}
-// BatchDeliver sends concurrent POST requests. Returns an error if any of the
-// requests had an error.
+// BatchDeliver sends concurrent POST requests. Returns an error if any of the requests had an error.
func (h HttpSigTransport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
var wg sync.WaitGroup
errCh := make(chan error, len(recipients))
diff --git a/vendor/github.com/superseriousbusiness/activity/pub/util.go b/vendor/github.com/superseriousbusiness/activity/pub/util.go
index a0675b76e..e917205ee 100644
--- a/vendor/github.com/superseriousbusiness/activity/pub/util.go
+++ b/vendor/github.com/superseriousbusiness/activity/pub/util.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
"net/url"
"strings"
@@ -71,6 +72,59 @@ const (
acceptHeader = "Accept"
)
+// readActivityPubReq reads ActivityPub data from an incoming request, handling body close.
+func readActivityPubReq(req *http.Request) (map[string]interface{}, error) {
+ // Ensure closed when done.
+ defer req.Body.Close()
+
+ var m map[string]interface{}
+
+ // Wrap body in a JSON decoder.
+ dec := json.NewDecoder(req.Body)
+
+ // Decode JSON body as "raw" AP data map.
+ if err := dec.Decode(&m); err != nil {
+ return nil, err
+ }
+
+ // Perform a final second decode to ensure no trailing
+ // garbage data or second JSON value (indicates malformed).
+ if err := dec.Decode(&struct{}{}); err != io.EOF {
+ return nil, errors.New("trailing data after json")
+ }
+
+ return m, nil
+}
+
+// readActivityPubResp reads ActivityPub data from a dereference response, handling media type check and body close.
+func readActivityPubResp(resp *http.Response) (map[string]interface{}, error) {
+ // Ensure closed when done.
+ defer resp.Body.Close()
+
+ // Check the incoming response media type is the expected ActivityPub content-type.
+ if mediaType := resp.Header.Get("Content-Type"); !headerIsActivityPubMediaType(mediaType) {
+ return nil, fmt.Errorf("%s %s resp was not ActivityPub media type: %s", resp.Request.Method, resp.Request.URL, mediaType)
+ }
+
+ var m map[string]interface{}
+
+ // Wrap body in a JSON decoder.
+ dec := json.NewDecoder(resp.Body)
+
+ // Decode JSON body as "raw" AP data map.
+ if err := dec.Decode(&m); err != nil {
+ return nil, err
+ }
+
+ // Perform a final second decode to ensure no trailing
+ // garbage data or second JSON value (indicates malformed).
+ if err := dec.Decode(&struct{}{}); err != io.EOF {
+ return nil, errors.New("trailing data after json")
+ }
+
+ return m, nil
+}
+
// isActivityPubPost returns true if the request is a POST request that has the
// ActivityStreams content type header
func isActivityPubPost(r *http.Request) bool {
@@ -774,12 +828,12 @@ func mustHaveActivityActorsMatchObjectActors(c context.Context,
if err != nil {
return err
}
- b, err := tport.Dereference(c, iri)
+ resp, err := tport.Dereference(c, iri)
if err != nil {
return err
}
- var m map[string]interface{}
- if err = json.Unmarshal(b, &m); err != nil {
+ m, err := readActivityPubResp(resp)
+ if err != nil {
return err
}
t, err := streams.ToType(c, m)
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 00ee7a1a5..a2f5ce0ea 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -507,7 +507,7 @@ github.com/stretchr/testify/suite
# github.com/subosito/gotenv v1.6.0
## explicit; go 1.18
github.com/subosito/gotenv
-# github.com/superseriousbusiness/activity v1.6.0-gts
+# github.com/superseriousbusiness/activity v1.6.0-gts.0.20240221151241-5d56c04088d4
## explicit; go 1.18
github.com/superseriousbusiness/activity/pub
github.com/superseriousbusiness/activity/streams