[bugfix] Fix remaining mangled URI escaping issues in statuses + accounts (#1712)

* start fiddling with normalize + extract functions

* normalize attachment name (image description)

* NormalizeAccountableSummary

* normalize summary + name

internal/ap/extract.go

@@ -56,10 +56,13 @@ func ExtractName(i WithName) string {
 		return ""
 	}
 
-	// take the first name string we can find
+	// Take the first useful value for the name string we can find.
 	for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() {
-		if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
+		switch {
+		case iter.IsXMLSchemaString():
 			return iter.GetXMLSchemaString()
+		case iter.IsIRI():
+			return iter.GetIRI().String()
 		}
 	}
 
@@ -253,10 +256,10 @@ func ExtractSummary(i WithSummary) string {
 
 	for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() {
 		switch {
-		case iter.IsIRI():
-			return iter.GetIRI().String()
 		case iter.IsXMLSchemaString():
 			return iter.GetXMLSchemaString()
+		case iter.IsIRI():
+			return iter.GetIRI().String()
 		}
 	}
 
@@ -354,10 +357,10 @@ func ExtractContent(i WithContent) string {
 	}
 
 	for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() {
-		if iter.IsXMLSchemaString() {
+		switch {
+		case iter.IsXMLSchemaString():
 			return iter.GetXMLSchemaString()
-		}
+		case iter.IsIRI():
-		if iter.IsIRI() && iter.GetIRI() != nil {
 			return iter.GetIRI().String()
 		}
 	}

internal/ap/interfaces.go

@@ -30,6 +30,7 @@ type Accountable interface {
 	WithName
 	WithImage
 	WithSummary
+	WithSetSummary
 	WithDiscoverable
 	WithURL
 	WithPublicKey
@@ -50,7 +51,9 @@ type Statusable interface {
 	WithTypeName
 
 	WithSummary
+	WithSetSummary
 	WithName
+	WithSetName
 	WithInReplyTo
 	WithPublished
 	WithURL
@@ -73,6 +76,7 @@ type Attachmentable interface {
 	WithMediaType
 	WithURL
 	WithName
+	WithSetName
 	WithBlurhash
 }
 
@@ -193,6 +197,11 @@ type WithName interface {
 	GetActivityStreamsName() vocab.ActivityStreamsNameProperty
 }
 
+// WithSetName represents an activity with a settable ActivityStreamsNameProperty
+type WithSetName interface {
+	SetActivityStreamsName(vocab.ActivityStreamsNameProperty)
+}
+
 // WithImage represents an activity with ActivityStreamsImageProperty
 type WithImage interface {
 	GetActivityStreamsImage() vocab.ActivityStreamsImageProperty
@@ -203,6 +212,11 @@ type WithSummary interface {
 	GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty
 }
 
+// WithSetSummary represents an activity that can have summary set on it.
+type WithSetSummary interface {
+	SetActivityStreamsSummary(vocab.ActivityStreamsSummaryProperty)
+}
+
 // WithDiscoverable represents an activity with TootDiscoverableProperty
 type WithDiscoverable interface {
 	GetTootDiscoverable() vocab.TootDiscoverableProperty

internal/ap/normalize.go

@@ -27,7 +27,7 @@ import (
 // The rawActivity map should the freshly deserialized json representation of the Activity.
 //
 // This function is a noop if the type passed in is anything except a Create with a Statusable as its Object.
-func NormalizeActivityObject(activity pub.Activity, rawActivity map[string]interface{}) {
+func NormalizeActivityObject(activity pub.Activity, rawJSON map[string]interface{}) {
 	if activity.GetTypeName() != ActivityCreate {
 		// Only interested in Create right now.
 		return
@@ -72,45 +72,157 @@ func NormalizeActivityObject(activity pub.Activity, rawActivity map[string]inter
 		return
 	}
 
-	object, ok := rawActivity["object"]
+	rawObject, ok := rawJSON["object"]
 	if !ok {
 		// No object in raw map.
 		return
 	}
 
-	rawStatusable, ok := object.(map[string]interface{})
+	rawStatusableJSON, ok := rawObject.(map[string]interface{})
 	if !ok {
 		// Object wasn't a json object.
 		return
 	}
 
-	// Pass in the statusable and its raw JSON representation.
+	// Normalize everything we can on the statusable.
-	NormalizeStatusableContent(statusable, rawStatusable)
+	NormalizeContent(statusable, rawStatusableJSON)
+	NormalizeAttachments(statusable, rawStatusableJSON)
+	NormalizeSummary(statusable, rawStatusableJSON)
+	NormalizeName(statusable, rawStatusableJSON)
 }
 
-// NormalizeStatusableContent replaces the Content of the given statusable
+// NormalizeContent replaces the Content of the given item
-// with the raw 'content' value from the given json object map.
+// with the raw 'content' value from the raw json object map.
 //
-// noop if there was no content in the json object map or the content was
+// noop if there was no content in the json object map or the
-// not a plain string.
+// content was not a plain string.
-func NormalizeStatusableContent(statusable Statusable, rawStatusable map[string]interface{}) {
+func NormalizeContent(item WithSetContent, rawJSON map[string]interface{}) {
-	content, ok := rawStatusable["content"]
+	rawContent, ok := rawJSON["content"]
 	if !ok {
-		// No content in rawStatusable.
+		// No content in rawJSON.
 		// TODO: In future we might also
 		// look for "contentMap" property.
 		return
 	}
 
-	rawContent, ok := content.(string)
+	content, ok := rawContent.(string)
 	if !ok {
 		// Not interested in content arrays.
 		return
 	}
 
-	// Set normalized content property from the raw string; this
+	// Set normalized content property from the raw string;
-	// will replace any existing content property on the statusable.
+	// this replaces any existing content property on the item.
 	contentProp := streams.NewActivityStreamsContentProperty()
-	contentProp.AppendXMLSchemaString(rawContent)
+	contentProp.AppendXMLSchemaString(content)
-	statusable.SetActivityStreamsContent(contentProp)
+	item.SetActivityStreamsContent(contentProp)
+}
+
+// NormalizeAttachments normalizes all attachments (if any) of the given
+// itm, replacing the 'name' (aka content warning) field of each attachment
+// with the raw 'name' value from the raw json object map.
+//
+// noop if there are no attachments; noop if attachment is not a format
+// we can understand.
+func NormalizeAttachments(item WithAttachment, rawJSON map[string]interface{}) {
+	rawAttachments, ok := rawJSON["attachment"]
+	if !ok {
+		// No attachments in rawJSON.
+		return
+	}
+
+	// Convert to slice if not already,
+	// so we can iterate through it.
+	var attachments []interface{}
+	if attachments, ok = rawAttachments.([]interface{}); !ok {
+		attachments = []interface{}{rawAttachments}
+	}
+
+	attachmentProperty := item.GetActivityStreamsAttachment()
+	if attachmentProperty == nil {
+		// Nothing to do here.
+		return
+	}
+
+	if l := attachmentProperty.Len(); l == 0 || l != len(attachments) {
+		// Mismatch between item and
+		// JSON, can't normalize.
+		return
+	}
+
+	// Keep an index of where we are in the iter;
+	// we need this so we can modify the correct
+	// attachment, in case of multiples.
+	i := -1
+
+	for iter := attachmentProperty.Begin(); iter != attachmentProperty.End(); iter = iter.Next() {
+		i++
+
+		t := iter.GetType()
+		if t == nil {
+			continue
+		}
+
+		attachmentable, ok := t.(Attachmentable)
+		if !ok {
+			continue
+		}
+
+		rawAttachment, ok := attachments[i].(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		NormalizeName(attachmentable, rawAttachment)
+	}
+}
+
+// NormalizeSummary replaces the Summary of the given item
+// with the raw 'summary' value from the raw json object map.
+//
+// noop if there was no summary in the json object map or the
+// summary was not a plain string.
+func NormalizeSummary(item WithSetSummary, rawJSON map[string]interface{}) {
+	rawSummary, ok := rawJSON["summary"]
+	if !ok {
+		// No summary in rawJSON.
+		return
+	}
+
+	summary, ok := rawSummary.(string)
+	if !ok {
+		// Not interested in non-string summary.
+		return
+	}
+
+	// Set normalized summary property from the raw string; this
+	// will replace any existing summary property on the item.
+	summaryProp := streams.NewActivityStreamsSummaryProperty()
+	summaryProp.AppendXMLSchemaString(summary)
+	item.SetActivityStreamsSummary(summaryProp)
+}
+
+// NormalizeName replaces the Name of the given item
+// with the raw 'name' value from the raw json object map.
+//
+// noop if there was no name in the json object map or the
+// name was not a plain string.
+func NormalizeName(item WithSetName, rawJSON map[string]interface{}) {
+	rawName, ok := rawJSON["name"]
+	if !ok {
+		// No name in rawJSON.
+		return
+	}
+
+	name, ok := rawName.(string)
+	if !ok {
+		// Not interested in non-string name.
+		return
+	}
+
+	// Set normalized name property from the raw string; this
+	// will replace any existing name property on the item.
+	nameProp := streams.NewActivityStreamsNameProperty()
+	nameProp.AppendXMLSchemaString(name)
+	item.SetActivityStreamsName(nameProp)
 }

internal/ap/normalize_test.go

@@ -33,8 +33,37 @@ type NormalizeTestSuite struct {
 	suite.Suite
 }
 
-func (suite *NormalizeTestSuite) GetStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) {
+func (suite *NormalizeTestSuite) jsonToType(rawJson string) (vocab.Type, map[string]interface{}) {
-	rawJson := `{
+	var raw map[string]interface{}
+	err := json.Unmarshal([]byte(rawJson), &raw)
+	if err != nil {
+		panic(err)
+	}
+
+	t, err := streams.ToType(context.Background(), raw)
+	if err != nil {
+		panic(err)
+	}
+
+	return t, raw
+}
+
+func (suite *NormalizeTestSuite) typeToJson(t vocab.Type) string {
+	m, err := streams.Serialize(t)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	b, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	return string(b)
+}
+
+func (suite *NormalizeTestSuite) getStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) {
+	t, raw := suite.jsonToType(`{
 		"@context": [
 		  "https://www.w3.org/ns/activitystreams",
 		  "https://example.org/schemas/litepub-0.1.jsonld",
@@ -74,24 +103,117 @@ func (suite *NormalizeTestSuite) GetStatusable() (vocab.ActivityStreamsNote, map
 		  "https://www.w3.org/ns/activitystreams#Public"
 		],
 		"type": "Note"
-	  }`
+	  }`)
 
-	var rawNote map[string]interface{}
+	return t.(vocab.ActivityStreamsNote), raw
-	err := json.Unmarshal([]byte(rawJson), &rawNote)
-	if err != nil {
-		panic(err)
 }
 
-	t, err := streams.ToType(context.Background(), rawNote)
+func (suite *NormalizeTestSuite) getStatusableWithOneAttachment() (vocab.ActivityStreamsNote, map[string]interface{}) {
-	if err != nil {
+	t, raw := suite.jsonToType(`{
-		panic(err)
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+		"type": "Note",
+		"url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ",
+		"attributedTo": "https://example.org/users/hourlycatbot",
+		"to": "https://www.w3.org/ns/activitystreams#Public",
+		"attachment": [
+		  {
+			"type": "Document",
+			"mediaType": "image/jpeg",
+			"url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg",
+			"name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''"
+		  }
+		]
+	  }`)
+
+	return t.(vocab.ActivityStreamsNote), raw
 }
 
-	return t.(vocab.ActivityStreamsNote), rawNote
+func (suite *NormalizeTestSuite) getStatusableWithOneAttachmentEmbedded() (vocab.ActivityStreamsNote, map[string]interface{}) {
+	t, raw := suite.jsonToType(`{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+		"type": "Note",
+		"url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ",
+		"attributedTo": "https://example.org/users/hourlycatbot",
+		"to": "https://www.w3.org/ns/activitystreams#Public",
+		"attachment": {
+		  "type": "Document",
+		  "mediaType": "image/jpeg",
+		  "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg",
+		  "name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''"
+		}
+	  }`)
+
+	return t.(vocab.ActivityStreamsNote), raw
+}
+
+func (suite *NormalizeTestSuite) getStatusableWithMultipleAttachments() (vocab.ActivityStreamsNote, map[string]interface{}) {
+	t, raw := suite.jsonToType(`{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+		"type": "Note",
+		"url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ",
+		"attributedTo": "https://example.org/users/hourlycatbot",
+		"to": "https://www.w3.org/ns/activitystreams#Public",
+		"attachment": [
+		  {
+			"type": "Document",
+			"mediaType": "image/jpeg",
+			"url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg",
+			"name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''"
+		  },
+		  {
+			"type": "Document",
+			"mediaType": "image/jpeg",
+			"url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg",
+			"name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!"
+		  },
+		  {
+			"type": "Document",
+			"mediaType": "image/jpeg",
+			"url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+		  },
+		  {
+			"type": "Document",
+			"mediaType": "image/jpeg",
+			"url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg",
+			"name": "danger: #cute but will claw you :("
+		  }
+		]
+	  }`)
+
+	return t.(vocab.ActivityStreamsNote), raw
+}
+
+func (suite *NormalizeTestSuite) getStatusableWithWeirdSummaryAndName() (vocab.ActivityStreamsNote, map[string]interface{}) {
+	t, raw := suite.jsonToType(`{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+		"type": "Note",
+		"url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ",
+		"attributedTo": "https://example.org/users/hourlycatbot",
+		"to": "https://www.w3.org/ns/activitystreams#Public",
+		"summary": "warning: #WEIRD #SUMMARY ;;;;a;;a;asv    khop8273987(*^&^)",
+		"name": "WARNING: #WEIRD #nameEE ;;;;a;;a;asv    khop8273987(*^&^)"
+	  }`)
+
+	return t.(vocab.ActivityStreamsNote), raw
+}
+
+func (suite *NormalizeTestSuite) getAccountable() (vocab.ActivityStreamsPerson, map[string]interface{}) {
+	t, raw := suite.jsonToType(`{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/someone",
+		"summary": "about: I'm a #Barbie #girl in a #Barbie #world\nLife in plastic, it's fantastic\nYou can brush my hair, undress me everywhere\nImagination, life is your creation\nI'm a blonde bimbo girl\nIn a fantasy world\nDress me up, make it tight\nI'm your dolly\nYou're my doll, rock and roll\nFeel the glamour in pink\nKiss me here, touch me there\nHanky panky",
+		"type": "Person"
+	  }`)
+
+	return t.(vocab.ActivityStreamsPerson), raw
 }
 
 func (suite *NormalizeTestSuite) TestNormalizeActivityObject() {
-	note, rawNote := suite.GetStatusable()
+	note, rawNote := suite.getStatusable()
 	suite.Equal(`update: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration%3C/a%3E.%3Cbr%3E%3Cbr%3EIn%20fact,%20100,000%20new%20accounts%20have%20been%20created%20since%20last%20night.%3Cbr%3E%3Cbr%3ESince%20last%20night&%2339;s%20spike%208,000-12,000%20new%20accounts%20are%20being%20created%20every%20hour.%3Cbr%3E%3Cbr%3EYesterday,%20I%20estimated%20that%20Mastodon%20would%20have%208%20million%20users%20by%20the%20end%20of%20the%20week.%20That%20might%20happen%20a%20lot%20sooner%20if%20this%20trend%20continues.`, ap.ExtractContent(note))
 
 	create := testrig.WrapAPNoteInCreate(
@@ -105,6 +227,202 @@ func (suite *NormalizeTestSuite) TestNormalizeActivityObject() {
 	suite.Equal(`UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration</a>.<br><br>In fact, 100,000 new accounts have been created since last night.<br><br>Since last night&#39;s spike 8,000-12,000 new accounts are being created every hour.<br><br>Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.`, ap.ExtractContent(note))
 }
 
+func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachment() {
+	note, rawNote := suite.getStatusableWithOneAttachment()
+
+	// Without normalization, the 'name' field of
+	// the attachment(s) should be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": {
+    "mediaType": "image/jpeg",
+    "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27",
+    "type": "Document",
+    "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+  },
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+
+	// Normalize it!
+	ap.NormalizeAttachments(note, rawNote)
+
+	// After normalization, the 'name' field of the
+	// attachment should no longer be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": {
+    "mediaType": "image/jpeg",
+    "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''",
+    "type": "Document",
+    "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+  },
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachmentEmbedded() {
+	note, rawNote := suite.getStatusableWithOneAttachmentEmbedded()
+
+	// Without normalization, the 'name' field of
+	// the attachment(s) should be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": {
+    "mediaType": "image/jpeg",
+    "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27",
+    "type": "Document",
+    "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+  },
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+
+	// Normalize it!
+	ap.NormalizeAttachments(note, rawNote)
+
+	// After normalization, the 'name' field of the
+	// attachment should no longer be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": {
+    "mediaType": "image/jpeg",
+    "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''",
+    "type": "Document",
+    "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+  },
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsMultipleAttachments() {
+	note, rawNote := suite.getStatusableWithMultipleAttachments()
+
+	// Without normalization, the 'name' field of
+	// the attachment(s) should be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": [
+    {
+      "mediaType": "image/jpeg",
+      "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "name": "hello: here's another #picture%20%23of%20%23a%20%23cat,%20hope%20you%20like%20it!!!!!!!",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "name": "danger: #cute%20but%20will%20claw%20you%20:(",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    }
+  ],
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+
+	// Normalize it!
+	ap.NormalizeAttachments(note, rawNote)
+
+	// After normalization, the 'name' field of the
+	// attachment should no longer be all jacked up.
+	suite.Equal(`{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "attachment": [
+    {
+      "mediaType": "image/jpeg",
+      "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    },
+    {
+      "mediaType": "image/jpeg",
+      "name": "danger: #cute but will claw you :(",
+      "type": "Document",
+      "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg"
+    }
+  ],
+  "attributedTo": "https://example.org/users/hourlycatbot",
+  "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ",
+  "to": "https://www.w3.org/ns/activitystreams#Public",
+  "type": "Note",
+  "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ"
+}`, suite.typeToJson(note))
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeAccountableSummary() {
+	accountable, rawAccount := suite.getAccountable()
+	suite.Equal(`about: I'm a #Barbie%20%23girl%20in%20a%20%23Barbie%20%23world%0ALife%20in%20plastic,%20it%27s%20fantastic%0AYou%20can%20brush%20my%20hair,%20undress%20me%20everywhere%0AImagination,%20life%20is%20your%20creation%0AI%27m%20a%20blonde%20bimbo%20girl%0AIn%20a%20fantasy%20world%0ADress%20me%20up,%20make%20it%20tight%0AI%27m%20your%20dolly%0AYou%27re%20my%20doll,%20rock%20and%20roll%0AFeel%20the%20glamour%20in%20pink%0AKiss%20me%20here,%20touch%20me%20there%0AHanky%20panky`, ap.ExtractSummary(accountable))
+
+	ap.NormalizeSummary(accountable, rawAccount)
+	suite.Equal(`about: I'm a #Barbie #girl in a #Barbie #world
+Life in plastic, it's fantastic
+You can brush my hair, undress me everywhere
+Imagination, life is your creation
+I'm a blonde bimbo girl
+In a fantasy world
+Dress me up, make it tight
+I'm your dolly
+You're my doll, rock and roll
+Feel the glamour in pink
+Kiss me here, touch me there
+Hanky panky`, ap.ExtractSummary(accountable))
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeStatusableSummary() {
+	statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName()
+	suite.Equal(`warning: #WEIRD%20%23SUMMARY%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractSummary(statusable))
+
+	ap.NormalizeSummary(statusable, rawAccount)
+	suite.Equal(`warning: #WEIRD #SUMMARY ;;;;a;;a;asv    khop8273987(*^&^)`, ap.ExtractSummary(statusable))
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeStatusableName() {
+	statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName()
+	suite.Equal(`warning: #WEIRD%20%23nameEE%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractName(statusable))
+
+	ap.NormalizeName(statusable, rawAccount)
+	suite.Equal(`WARNING: #WEIRD #nameEE ;;;;a;;a;asv    khop8273987(*^&^)`, ap.ExtractName(statusable))
+}
+
 func TestNormalizeTestSuite(t *testing.T) {
 	suite.Run(t, new(NormalizeTestSuite))
 }

internal/ap/resolve.go

@@ -27,8 +27,7 @@ import (
 )
 
 // ResolveStatusable tries to resolve the given bytes into an ActivityPub Statusable representation.
-// It will then perform normalization on the Statusable by calling NormalizeStatusable, so that
+// It will then perform normalization on the Statusable.
-// callers don't need to bother doing extra steps.
 //
 // Works for: Article, Document, Image, Video, Note, Page, Event, Place, Profile
 func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) {
@@ -73,11 +72,16 @@ func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) {
 		return nil, newErrWrongType(err)
 	}
 
-	NormalizeStatusableContent(statusable, rawStatusable)
+	NormalizeContent(statusable, rawStatusable)
+	NormalizeAttachments(statusable, rawStatusable)
+	NormalizeSummary(statusable, rawStatusable)
+	NormalizeName(statusable, rawStatusable)
+
 	return statusable, nil
 }
 
 // ResolveStatusable tries to resolve the given bytes into an ActivityPub Accountable representation.
+// It will then perform normalization on the Accountable.
 //
 // Works for: Application, Group, Organization, Person, Service
 func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) {
@@ -114,5 +118,7 @@ func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) {
 		return nil, newErrWrongType(err)
 	}
 
+	NormalizeSummary(accountable, rawAccountable)
+
 	return accountable, nil
 }

internal/processing/fromfederator.go

@@ -373,15 +373,11 @@ func (p *Processor) processUpdateAccountFromFederator(ctx context.Context, feder
 		return errors.New("profile was not parseable as *gtsmodel.Account")
 	}
 
-	incomingAccountURL, err := url.Parse(incomingAccount.URI)
+	// Call UpdateAccount with force to reflect that
-	if err != nil {
+	// we want to fetch new bio, avatar, header, etc.
-		return err
+	if _, err := p.federator.UpdateAccount(ctx,
-	}
-
-	// further database updates occur inside getremoteaccount
-	if _, err := p.federator.GetAccountByURI(ctx,
 		federatorMsg.ReceivingAccount.Username,
-		incomingAccountURL,
+		incomingAccount,
 		true,
 	); err != nil {
 		return fmt.Errorf("error enriching updated account from federator: %s", err)