From f559d4626118c103225134b4fa87a075702c7e41 Mon Sep 17 00:00:00 2001
From: Sam Lade <sam@sentynel.com>
Date: Mon, 20 Feb 2023 15:27:41 +0000
Subject: [PATCH] [bugfix] Fix failure to look up remote profiles with
 duplicate emojis in some cases (#1534)

* Tidy up emoji parsing on profile submission

Don't bother reparsing for emoji unless one of the fields that can have
emoji in it has changed.
Deduplicate emoji between the display name and profile note - I'm not
sure whether this was hurting anything, but better safe.

* Deduplicate emoji when parsing remote accounts

Some servers - Misskey at least - don't deduplicate emoji, so it's
possible to get an account which has the same emoji used in both the
display name and note and therefore includes that emoji twice in its
metadata. When we start trying to put those into our database, we run
into a uniqueness constraint and fall over.

This change just deduplicates at the point of construction of an
account.
---
 internal/ap/extract.go                |  4 ++
 internal/processing/account/update.go | 58 +++++++++++++++------------
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/internal/ap/extract.go b/internal/ap/extract.go
index f3ff6d3b2..5d8dbd49d 100644
--- a/internal/ap/extract.go
+++ b/internal/ap/extract.go
@@ -453,6 +453,7 @@ func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 // ExtractEmojis returns a slice of emojis on the interface.
 func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) {
 	emojis := []*gtsmodel.Emoji{}
+	emojiMap := make(map[string]*gtsmodel.Emoji)
 	tagsProp := i.GetActivityStreamsTag()
 	if tagsProp == nil {
 		return emojis, nil
@@ -477,6 +478,9 @@ func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) {
 			continue
 		}
 
+		emojiMap[emoji.URI] = emoji
+	}
+	for _, emoji := range emojiMap {
 		emojis = append(emojis, emoji)
 	}
 	return emojis, nil
diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go
index 675793800..e6867bfd3 100644
--- a/internal/processing/account/update.go
+++ b/internal/processing/account/update.go
@@ -45,23 +45,14 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
 		account.Bot = form.Bot
 	}
 
-	account.Emojis = []*gtsmodel.Emoji{}
-	account.EmojiIDs = []string{}
+	reparseEmojis := false
 
 	if form.DisplayName != nil {
 		if err := validate.DisplayName(*form.DisplayName); err != nil {
 			return nil, gtserror.NewErrorBadRequest(err)
 		}
 		account.DisplayName = text.SanitizePlaintext(*form.DisplayName)
-	}
-
-	// Re-parse for emojis regardless of whether the DisplayName changed
-	// because we can't otherwise tell which emojis belong to DisplayName
-	// and which belong to Note
-	formatResult := p.formatter.FromPlainEmojiOnly(ctx, p.parseMention, account.ID, "", account.DisplayName)
-	for _, emoji := range formatResult.Emojis {
-		account.Emojis = append(account.Emojis, emoji)
-		account.EmojiIDs = append(account.EmojiIDs, emoji.ID)
+		reparseEmojis = true
 	}
 
 	if form.Note != nil {
@@ -71,23 +62,40 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
 
 		// Set the raw note before processing
 		account.NoteRaw = *form.Note
+		reparseEmojis = true
 	}
 
-	// As per DisplayName, we need to reparse regardless to keep emojis straight
-	// Process note to generate a valid HTML representation
-	var f text.FormatFunc
-	if account.StatusFormat == "markdown" {
-		f = p.formatter.FromMarkdown
-	} else {
-		f = p.formatter.FromPlain
-	}
-	formatted := f(ctx, p.parseMention, account.ID, "", account.NoteRaw)
+	if reparseEmojis {
+		// If either DisplayName or Note changed, reparse both, because we
+		// can't otherwise tell which one each emoji belongs to.
+		// Deduplicate emojis between the two fields.
+		emojis := make(map[string]*gtsmodel.Emoji)
+		formatResult := p.formatter.FromPlainEmojiOnly(ctx, p.parseMention, account.ID, "", account.DisplayName)
+		for _, emoji := range formatResult.Emojis {
+			emojis[emoji.ID] = emoji
+		}
 
-	// Set updated HTML-ified note
-	account.Note = formatted.HTML
-	for _, emoji := range formatted.Emojis {
-		account.Emojis = append(account.Emojis, emoji)
-		account.EmojiIDs = append(account.EmojiIDs, emoji.ID)
+		// Process note to generate a valid HTML representation
+		var f text.FormatFunc
+		if account.StatusFormat == "markdown" {
+			f = p.formatter.FromMarkdown
+		} else {
+			f = p.formatter.FromPlain
+		}
+		formatted := f(ctx, p.parseMention, account.ID, "", account.NoteRaw)
+
+		// Set updated HTML-ified note
+		account.Note = formatted.HTML
+		for _, emoji := range formatted.Emojis {
+			emojis[emoji.ID] = emoji
+		}
+
+		account.Emojis = []*gtsmodel.Emoji{}
+		account.EmojiIDs = []string{}
+		for eid, emoji := range emojis {
+			account.Emojis = append(account.Emojis, emoji)
+			account.EmojiIDs = append(account.EmojiIDs, eid)
+		}
 	}
 
 	if form.Avatar != nil && form.Avatar.Size != 0 {