[performance] add caching of status fave, boost of, in reply to ID lists (#2060)

This commit is contained in:
kim 2023-08-04 12:28:33 +01:00 committed by GitHub
parent 00adf18c24
commit 9a291dea84
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 610 additions and 406 deletions

2
go.mod
View file

@ -5,7 +5,7 @@ go 1.20
require ( require (
codeberg.org/gruf/go-bytesize v1.0.2 codeberg.org/gruf/go-bytesize v1.0.2
codeberg.org/gruf/go-byteutil v1.1.2 codeberg.org/gruf/go-byteutil v1.1.2
codeberg.org/gruf/go-cache/v3 v3.5.3 codeberg.org/gruf/go-cache/v3 v3.5.5
codeberg.org/gruf/go-debug v1.3.0 codeberg.org/gruf/go-debug v1.3.0
codeberg.org/gruf/go-errors/v2 v2.2.0 codeberg.org/gruf/go-errors/v2 v2.2.0
codeberg.org/gruf/go-fastcopy v1.1.2 codeberg.org/gruf/go-fastcopy v1.1.2

4
go.sum
View file

@ -48,8 +48,8 @@ codeberg.org/gruf/go-bytesize v1.0.2/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacp
codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU= codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
codeberg.org/gruf/go-byteutil v1.1.2 h1:TQLZtTxTNca9xEfDIndmo7nBYxeS94nrv/9DS3Nk5Tw= codeberg.org/gruf/go-byteutil v1.1.2 h1:TQLZtTxTNca9xEfDIndmo7nBYxeS94nrv/9DS3Nk5Tw=
codeberg.org/gruf/go-byteutil v1.1.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU= codeberg.org/gruf/go-byteutil v1.1.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
codeberg.org/gruf/go-cache/v3 v3.5.3 h1:CRO2syVQxT/JbqDnUxzjeJkLInihEmTlJOkrOgkTmqI= codeberg.org/gruf/go-cache/v3 v3.5.5 h1:Ce7odyvr8oF6h49LSjPL7AZs2QGyKMN9BPkgKcfR0BA=
codeberg.org/gruf/go-cache/v3 v3.5.3/go.mod h1:NbsGQUgEdNFd631WSasvCHIVAaY9ovuiSeoBwtsIeDc= codeberg.org/gruf/go-cache/v3 v3.5.5/go.mod h1:NbsGQUgEdNFd631WSasvCHIVAaY9ovuiSeoBwtsIeDc=
codeberg.org/gruf/go-debug v1.3.0 h1:PIRxQiWUFKtGOGZFdZ3Y0pqyfI0Xr87j224IYe2snZs= codeberg.org/gruf/go-debug v1.3.0 h1:PIRxQiWUFKtGOGZFdZ3Y0pqyfI0Xr87j224IYe2snZs=
codeberg.org/gruf/go-debug v1.3.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg= codeberg.org/gruf/go-debug v1.3.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg=
codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4= codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=

View file

@ -196,6 +196,21 @@ func (c *Caches) setuphooks() {
// c.GTS.Media().Invalidate("StatusID") will not work. // c.GTS.Media().Invalidate("StatusID") will not work.
c.GTS.Media().Invalidate("ID", id) c.GTS.Media().Invalidate("ID", id)
} }
if status.BoostOfID != "" {
// Invalidate boost ID list of the original status.
c.GTS.BoostOfIDs().Invalidate(status.BoostOfID)
}
if status.InReplyToID != "" {
// Invalidate in reply to ID list of original status.
c.GTS.InReplyToIDs().Invalidate(status.InReplyToID)
}
})
c.GTS.StatusFave().SetInvalidateCallback(func(fave *gtsmodel.StatusFave) {
// Invalidate status fave ID list for this status.
c.GTS.StatusFaveIDs().Invalidate(fave.StatusID)
}) })
c.GTS.User().SetInvalidateCallback(func(user *gtsmodel.User) { c.GTS.User().SetInvalidateCallback(func(user *gtsmodel.User) {

69
internal/cache/gts.go vendored
View file

@ -34,6 +34,7 @@ type GTSCaches struct {
accountNote *result.Cache[*gtsmodel.AccountNote] accountNote *result.Cache[*gtsmodel.AccountNote]
block *result.Cache[*gtsmodel.Block] block *result.Cache[*gtsmodel.Block]
blockIDs *SliceCache[string] blockIDs *SliceCache[string]
boostOfIDs *SliceCache[string]
domainBlock *domain.BlockCache domainBlock *domain.BlockCache
emoji *result.Cache[*gtsmodel.Emoji] emoji *result.Cache[*gtsmodel.Emoji]
emojiCategory *result.Cache[*gtsmodel.EmojiCategory] emojiCategory *result.Cache[*gtsmodel.EmojiCategory]
@ -42,6 +43,7 @@ type GTSCaches struct {
followRequest *result.Cache[*gtsmodel.FollowRequest] followRequest *result.Cache[*gtsmodel.FollowRequest]
followRequestIDs *SliceCache[string] followRequestIDs *SliceCache[string]
instance *result.Cache[*gtsmodel.Instance] instance *result.Cache[*gtsmodel.Instance]
inReplyToIDs *SliceCache[string]
list *result.Cache[*gtsmodel.List] list *result.Cache[*gtsmodel.List]
listEntry *result.Cache[*gtsmodel.ListEntry] listEntry *result.Cache[*gtsmodel.ListEntry]
marker *result.Cache[*gtsmodel.Marker] marker *result.Cache[*gtsmodel.Marker]
@ -51,6 +53,7 @@ type GTSCaches struct {
report *result.Cache[*gtsmodel.Report] report *result.Cache[*gtsmodel.Report]
status *result.Cache[*gtsmodel.Status] status *result.Cache[*gtsmodel.Status]
statusFave *result.Cache[*gtsmodel.StatusFave] statusFave *result.Cache[*gtsmodel.StatusFave]
statusFaveIDs *SliceCache[string]
tag *result.Cache[*gtsmodel.Tag] tag *result.Cache[*gtsmodel.Tag]
tombstone *result.Cache[*gtsmodel.Tombstone] tombstone *result.Cache[*gtsmodel.Tombstone]
user *result.Cache[*gtsmodel.User] user *result.Cache[*gtsmodel.User]
@ -66,6 +69,7 @@ func (c *GTSCaches) Init() {
c.initAccountNote() c.initAccountNote()
c.initBlock() c.initBlock()
c.initBlockIDs() c.initBlockIDs()
c.initBoostOfIDs()
c.initDomainBlock() c.initDomainBlock()
c.initEmoji() c.initEmoji()
c.initEmojiCategory() c.initEmojiCategory()
@ -73,6 +77,7 @@ func (c *GTSCaches) Init() {
c.initFollowIDs() c.initFollowIDs()
c.initFollowRequest() c.initFollowRequest()
c.initFollowRequestIDs() c.initFollowRequestIDs()
c.initInReplyToIDs()
c.initInstance() c.initInstance()
c.initList() c.initList()
c.initListEntry() c.initListEntry()
@ -84,6 +89,7 @@ func (c *GTSCaches) Init() {
c.initStatus() c.initStatus()
c.initStatusFave() c.initStatusFave()
c.initTag() c.initTag()
c.initStatusFaveIDs()
c.initTombstone() c.initTombstone()
c.initUser() c.initUser()
c.initWebfinger() c.initWebfinger()
@ -121,6 +127,11 @@ func (c *GTSCaches) BlockIDs() *SliceCache[string] {
return c.blockIDs return c.blockIDs
} }
// BoostOfIDs provides access to the boost of IDs list database cache.
func (c *GTSCaches) BoostOfIDs() *SliceCache[string] {
return c.boostOfIDs
}
// DomainBlock provides access to the domain block database cache. // DomainBlock provides access to the domain block database cache.
func (c *GTSCaches) DomainBlock() *domain.BlockCache { func (c *GTSCaches) DomainBlock() *domain.BlockCache {
return c.domainBlock return c.domainBlock
@ -169,6 +180,11 @@ func (c *GTSCaches) Instance() *result.Cache[*gtsmodel.Instance] {
return c.instance return c.instance
} }
// InReplyToIDs provides access to the status in reply to IDs list database cache.
func (c *GTSCaches) InReplyToIDs() *SliceCache[string] {
return c.inReplyToIDs
}
// List provides access to the gtsmodel List database cache. // List provides access to the gtsmodel List database cache.
func (c *GTSCaches) List() *result.Cache[*gtsmodel.List] { func (c *GTSCaches) List() *result.Cache[*gtsmodel.List] {
return c.list return c.list
@ -219,6 +235,11 @@ func (c *GTSCaches) Tag() *result.Cache[*gtsmodel.Tag] {
return c.tag return c.tag
} }
// StatusFaveIDs provides access to the status fave IDs list database cache.
func (c *GTSCaches) StatusFaveIDs() *SliceCache[string] {
return c.statusFaveIDs
}
// Tombstone provides access to the gtsmodel Tombstone database cache. // Tombstone provides access to the gtsmodel Tombstone database cache.
func (c *GTSCaches) Tombstone() *result.Cache[*gtsmodel.Tombstone] { func (c *GTSCaches) Tombstone() *result.Cache[*gtsmodel.Tombstone] {
return c.tombstone return c.tombstone
@ -247,7 +268,7 @@ func (c *GTSCaches) initAccount() {
{Name: "ID"}, {Name: "ID"},
{Name: "URI"}, {Name: "URI"},
{Name: "URL"}, {Name: "URL"},
{Name: "Username.Domain"}, {Name: "Username.Domain", AllowZero: true /* domain can be zero i.e. "" */},
{Name: "PublicKeyURI"}, {Name: "PublicKeyURI"},
{Name: "InboxURI"}, {Name: "InboxURI"},
{Name: "OutboxURI"}, {Name: "OutboxURI"},
@ -320,6 +341,20 @@ func (c *GTSCaches) initBlockIDs() {
)} )}
} }
func (c *GTSCaches) initBoostOfIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheBoostOfIDsMemRatio(),
)
log.Infof(nil, "BoostofIDs cache size = %d", cap)
c.boostOfIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
cap,
)}
}
func (c *GTSCaches) initDomainBlock() { func (c *GTSCaches) initDomainBlock() {
c.domainBlock = new(domain.BlockCache) c.domainBlock = new(domain.BlockCache)
} }
@ -336,7 +371,7 @@ func (c *GTSCaches) initEmoji() {
c.emoji = result.New([]result.Lookup{ c.emoji = result.New([]result.Lookup{
{Name: "ID"}, {Name: "ID"},
{Name: "URI"}, {Name: "URI"},
{Name: "Shortcode.Domain"}, {Name: "Shortcode.Domain", AllowZero: true /* domain can be zero i.e. "" */},
{Name: "ImageStaticURL"}, {Name: "ImageStaticURL"},
{Name: "CategoryID", Multi: true}, {Name: "CategoryID", Multi: true},
}, func(e1 *gtsmodel.Emoji) *gtsmodel.Emoji { }, func(e1 *gtsmodel.Emoji) *gtsmodel.Emoji {
@ -445,6 +480,20 @@ func (c *GTSCaches) initFollowRequestIDs() {
)} )}
} }
func (c *GTSCaches) initInReplyToIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheInReplyToIDsMemRatio(),
)
log.Infof(nil, "InReplyTo IDs cache size = %d", cap)
c.inReplyToIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
cap,
)}
}
func (c *GTSCaches) initInstance() { func (c *GTSCaches) initInstance() {
// Calculate maximum cache size. // Calculate maximum cache size.
cap := calculateResultCacheMax( cap := calculateResultCacheMax(
@ -622,6 +671,7 @@ func (c *GTSCaches) initStatus() {
{Name: "ID"}, {Name: "ID"},
{Name: "URI"}, {Name: "URI"},
{Name: "URL"}, {Name: "URL"},
{Name: "BoostOfID.AccountID"},
}, func(s1 *gtsmodel.Status) *gtsmodel.Status { }, func(s1 *gtsmodel.Status) *gtsmodel.Status {
s2 := new(gtsmodel.Status) s2 := new(gtsmodel.Status)
*s2 = *s1 *s2 = *s1
@ -643,6 +693,7 @@ func (c *GTSCaches) initStatusFave() {
c.statusFave = result.New([]result.Lookup{ c.statusFave = result.New([]result.Lookup{
{Name: "ID"}, {Name: "ID"},
{Name: "AccountID.StatusID"}, {Name: "AccountID.StatusID"},
{Name: "StatusID", Multi: true},
}, func(f1 *gtsmodel.StatusFave) *gtsmodel.StatusFave { }, func(f1 *gtsmodel.StatusFave) *gtsmodel.StatusFave {
f2 := new(gtsmodel.StatusFave) f2 := new(gtsmodel.StatusFave)
*f2 = *f1 *f2 = *f1
@ -652,6 +703,20 @@ func (c *GTSCaches) initStatusFave() {
c.statusFave.IgnoreErrors(ignoreErrors) c.statusFave.IgnoreErrors(ignoreErrors)
} }
func (c *GTSCaches) initStatusFaveIDs() {
// Calculate maximum cache size.
cap := calculateSliceCacheMax(
config.GetCacheStatusFaveIDsMemRatio(),
)
log.Infof(nil, "StatusFave IDs cache size = %d", cap)
c.statusFaveIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
cap,
)}
}
func (c *GTSCaches) initTag() { func (c *GTSCaches) initTag() {
// Calculate maximum cache size. // Calculate maximum cache size.
cap := calculateResultCacheMax( cap := calculateResultCacheMax(

View file

@ -180,12 +180,14 @@ type CacheConfiguration struct {
AccountNoteMemRatio float64 `name:"account-note-mem-ratio"` AccountNoteMemRatio float64 `name:"account-note-mem-ratio"`
BlockMemRatio float64 `name:"block-mem-ratio"` BlockMemRatio float64 `name:"block-mem-ratio"`
BlockIDsMemRatio float64 `name:"block-mem-ratio"` BlockIDsMemRatio float64 `name:"block-mem-ratio"`
BoostOfIDsMemRatio float64 `name:"boost-of-ids-mem-ratio"`
EmojiMemRatio float64 `name:"emoji-mem-ratio"` EmojiMemRatio float64 `name:"emoji-mem-ratio"`
EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"` EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"`
FollowMemRatio float64 `name:"follow-mem-ratio"` FollowMemRatio float64 `name:"follow-mem-ratio"`
FollowIDsMemRatio float64 `name:"follow-ids-mem-ratio"` FollowIDsMemRatio float64 `name:"follow-ids-mem-ratio"`
FollowRequestMemRatio float64 `name:"follow-request-mem-ratio"` FollowRequestMemRatio float64 `name:"follow-request-mem-ratio"`
FollowRequestIDsMemRatio float64 `name:"follow-request-ids-mem-ratio"` FollowRequestIDsMemRatio float64 `name:"follow-request-ids-mem-ratio"`
InReplyToIDsMemRatio float64 `name:"in-reply-to-ids-mem-ratio"`
InstanceMemRatio float64 `name:"instance-mem-ratio"` InstanceMemRatio float64 `name:"instance-mem-ratio"`
ListMemRatio float64 `name:"list-mem-ratio"` ListMemRatio float64 `name:"list-mem-ratio"`
ListEntryMemRatio float64 `name:"list-entry-mem-ratio"` ListEntryMemRatio float64 `name:"list-entry-mem-ratio"`
@ -196,6 +198,7 @@ type CacheConfiguration struct {
ReportMemRatio float64 `name:"report-mem-ratio"` ReportMemRatio float64 `name:"report-mem-ratio"`
StatusMemRatio float64 `name:"status-mem-ratio"` StatusMemRatio float64 `name:"status-mem-ratio"`
StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"`
StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"`
TagMemRatio float64 `name:"tag-mem-ratio"` TagMemRatio float64 `name:"tag-mem-ratio"`
TombstoneMemRatio float64 `name:"tombstone-mem-ratio"` TombstoneMemRatio float64 `name:"tombstone-mem-ratio"`
UserMemRatio float64 `name:"user-mem-ratio"` UserMemRatio float64 `name:"user-mem-ratio"`

View file

@ -149,12 +149,14 @@ var Defaults = Configuration{
AccountNoteMemRatio: 0.1, AccountNoteMemRatio: 0.1,
BlockMemRatio: 3, BlockMemRatio: 3,
BlockIDsMemRatio: 3, BlockIDsMemRatio: 3,
BoostOfIDsMemRatio: 3,
EmojiMemRatio: 3, EmojiMemRatio: 3,
EmojiCategoryMemRatio: 0.1, EmojiCategoryMemRatio: 0.1,
FollowMemRatio: 4, FollowMemRatio: 4,
FollowIDsMemRatio: 4, FollowIDsMemRatio: 4,
FollowRequestMemRatio: 2, FollowRequestMemRatio: 2,
FollowRequestIDsMemRatio: 2, FollowRequestIDsMemRatio: 2,
InReplyToIDsMemRatio: 3,
InstanceMemRatio: 1, InstanceMemRatio: 1,
ListMemRatio: 3, ListMemRatio: 3,
ListEntryMemRatio: 3, ListEntryMemRatio: 3,
@ -165,6 +167,7 @@ var Defaults = Configuration{
ReportMemRatio: 1, ReportMemRatio: 1,
StatusMemRatio: 18, StatusMemRatio: 18,
StatusFaveMemRatio: 5, StatusFaveMemRatio: 5,
StatusFaveIDsMemRatio: 3,
TagMemRatio: 3, TagMemRatio: 3,
TombstoneMemRatio: 2, TombstoneMemRatio: 2,
UserMemRatio: 0.1, UserMemRatio: 0.1,

View file

@ -2549,6 +2549,31 @@ func GetCacheBlockIDsMemRatio() float64 { return global.GetCacheBlockIDsMemRatio
// SetCacheBlockIDsMemRatio safely sets the value for global configuration 'Cache.BlockIDsMemRatio' field // SetCacheBlockIDsMemRatio safely sets the value for global configuration 'Cache.BlockIDsMemRatio' field
func SetCacheBlockIDsMemRatio(v float64) { global.SetCacheBlockIDsMemRatio(v) } func SetCacheBlockIDsMemRatio(v float64) { global.SetCacheBlockIDsMemRatio(v) }
// GetCacheBoostOfIDsMemRatio safely fetches the Configuration value for state's 'Cache.BoostOfIDsMemRatio' field
func (st *ConfigState) GetCacheBoostOfIDsMemRatio() (v float64) {
st.mutex.RLock()
v = st.config.Cache.BoostOfIDsMemRatio
st.mutex.RUnlock()
return
}
// SetCacheBoostOfIDsMemRatio safely sets the Configuration value for state's 'Cache.BoostOfIDsMemRatio' field
func (st *ConfigState) SetCacheBoostOfIDsMemRatio(v float64) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.BoostOfIDsMemRatio = v
st.reloadToViper()
}
// CacheBoostOfIDsMemRatioFlag returns the flag name for the 'Cache.BoostOfIDsMemRatio' field
func CacheBoostOfIDsMemRatioFlag() string { return "cache-boost-of-ids-mem-ratio" }
// GetCacheBoostOfIDsMemRatio safely fetches the value for global configuration 'Cache.BoostOfIDsMemRatio' field
func GetCacheBoostOfIDsMemRatio() float64 { return global.GetCacheBoostOfIDsMemRatio() }
// SetCacheBoostOfIDsMemRatio safely sets the value for global configuration 'Cache.BoostOfIDsMemRatio' field
func SetCacheBoostOfIDsMemRatio(v float64) { global.SetCacheBoostOfIDsMemRatio(v) }
// GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field // GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field
func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) { func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) {
st.mutex.RLock() st.mutex.RLock()
@ -2699,6 +2724,31 @@ func GetCacheFollowRequestIDsMemRatio() float64 { return global.GetCacheFollowRe
// SetCacheFollowRequestIDsMemRatio safely sets the value for global configuration 'Cache.FollowRequestIDsMemRatio' field // SetCacheFollowRequestIDsMemRatio safely sets the value for global configuration 'Cache.FollowRequestIDsMemRatio' field
func SetCacheFollowRequestIDsMemRatio(v float64) { global.SetCacheFollowRequestIDsMemRatio(v) } func SetCacheFollowRequestIDsMemRatio(v float64) { global.SetCacheFollowRequestIDsMemRatio(v) }
// GetCacheInReplyToIDsMemRatio safely fetches the Configuration value for state's 'Cache.InReplyToIDsMemRatio' field
func (st *ConfigState) GetCacheInReplyToIDsMemRatio() (v float64) {
st.mutex.RLock()
v = st.config.Cache.InReplyToIDsMemRatio
st.mutex.RUnlock()
return
}
// SetCacheInReplyToIDsMemRatio safely sets the Configuration value for state's 'Cache.InReplyToIDsMemRatio' field
func (st *ConfigState) SetCacheInReplyToIDsMemRatio(v float64) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.InReplyToIDsMemRatio = v
st.reloadToViper()
}
// CacheInReplyToIDsMemRatioFlag returns the flag name for the 'Cache.InReplyToIDsMemRatio' field
func CacheInReplyToIDsMemRatioFlag() string { return "cache-in-reply-to-ids-mem-ratio" }
// GetCacheInReplyToIDsMemRatio safely fetches the value for global configuration 'Cache.InReplyToIDsMemRatio' field
func GetCacheInReplyToIDsMemRatio() float64 { return global.GetCacheInReplyToIDsMemRatio() }
// SetCacheInReplyToIDsMemRatio safely sets the value for global configuration 'Cache.InReplyToIDsMemRatio' field
func SetCacheInReplyToIDsMemRatio(v float64) { global.SetCacheInReplyToIDsMemRatio(v) }
// GetCacheInstanceMemRatio safely fetches the Configuration value for state's 'Cache.InstanceMemRatio' field // GetCacheInstanceMemRatio safely fetches the Configuration value for state's 'Cache.InstanceMemRatio' field
func (st *ConfigState) GetCacheInstanceMemRatio() (v float64) { func (st *ConfigState) GetCacheInstanceMemRatio() (v float64) {
st.mutex.RLock() st.mutex.RLock()
@ -2949,6 +2999,31 @@ func GetCacheStatusFaveMemRatio() float64 { return global.GetCacheStatusFaveMemR
// SetCacheStatusFaveMemRatio safely sets the value for global configuration 'Cache.StatusFaveMemRatio' field // SetCacheStatusFaveMemRatio safely sets the value for global configuration 'Cache.StatusFaveMemRatio' field
func SetCacheStatusFaveMemRatio(v float64) { global.SetCacheStatusFaveMemRatio(v) } func SetCacheStatusFaveMemRatio(v float64) { global.SetCacheStatusFaveMemRatio(v) }
// GetCacheStatusFaveIDsMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveIDsMemRatio' field
func (st *ConfigState) GetCacheStatusFaveIDsMemRatio() (v float64) {
st.mutex.RLock()
v = st.config.Cache.StatusFaveIDsMemRatio
st.mutex.RUnlock()
return
}
// SetCacheStatusFaveIDsMemRatio safely sets the Configuration value for state's 'Cache.StatusFaveIDsMemRatio' field
func (st *ConfigState) SetCacheStatusFaveIDsMemRatio(v float64) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.StatusFaveIDsMemRatio = v
st.reloadToViper()
}
// CacheStatusFaveIDsMemRatioFlag returns the flag name for the 'Cache.StatusFaveIDsMemRatio' field
func CacheStatusFaveIDsMemRatioFlag() string { return "cache-status-fave-ids-mem-ratio" }
// GetCacheStatusFaveIDsMemRatio safely fetches the value for global configuration 'Cache.StatusFaveIDsMemRatio' field
func GetCacheStatusFaveIDsMemRatio() float64 { return global.GetCacheStatusFaveIDsMemRatio() }
// SetCacheStatusFaveIDsMemRatio safely sets the value for global configuration 'Cache.StatusFaveIDsMemRatio' field
func SetCacheStatusFaveIDsMemRatio(v float64) { global.SetCacheStatusFaveIDsMemRatio(v) }
// GetCacheTagMemRatio safely fetches the Configuration value for state's 'Cache.TagMemRatio' field // GetCacheTagMemRatio safely fetches the Configuration value for state's 'Cache.TagMemRatio' field
func (st *ConfigState) GetCacheTagMemRatio() (v float64) { func (st *ConfigState) GetCacheTagMemRatio() (v float64) {
st.mutex.RLock() st.mutex.RLock()

View file

@ -20,7 +20,6 @@ package bundb
import ( import (
"container/list" "container/list"
"context" "context"
"database/sql"
"errors" "errors"
"time" "time"
@ -96,6 +95,26 @@ func (s *statusDB) GetStatusByURL(ctx context.Context, url string) (*gtsmodel.St
) )
} }
func (s *statusDB) GetStatusBoost(ctx context.Context, boostOfID string, byAccountID string) (*gtsmodel.Status, error) {
return s.getStatus(
ctx,
"BoostOfID.AccountID",
func(status *gtsmodel.Status) error {
return s.newStatusQ(status).
Where("status.boost_of_id = ?", boostOfID).
Where("status.account_id = ?", byAccountID).
// Our old code actually allowed a status to
// be boosted multiple times by the same author,
// so limit our query + order to fetch latest.
Order("status.id DESC"). // our IDs are timestamped
Limit(1).
Scan(ctx)
},
boostOfID, byAccountID,
)
}
func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Status) error, keyParts ...any) (*gtsmodel.Status, error) { func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Status) error, keyParts ...any) (*gtsmodel.Status, error) {
// Fetch status from database cache with loader callback // Fetch status from database cache with loader callback
status, err := s.state.Caches.GTS.Status().Load(lookup, func() (*gtsmodel.Status, error) { status, err := s.state.Caches.GTS.Status().Load(lookup, func() (*gtsmodel.Status, error) {
@ -245,11 +264,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
} }
} }
if err := errs.Combine(); err != nil { return errs.Combine()
return gtserror.Newf("%w", err)
}
return nil
} }
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error { func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error {
@ -506,25 +521,17 @@ func (s *statusDB) GetStatusChildren(ctx context.Context, status *gtsmodel.Statu
} }
func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) { func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) {
var childIDs []string childIDs, err := s.getStatusReplyIDs(ctx, status.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
q := s.db. log.Errorf(ctx, "error getting status %s children: %v", status.ID, err)
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Column("status.id").
Where("? = ?", bun.Ident("status.in_reply_to_id"), status.ID)
if minID != "" {
q = q.Where("? > ?", bun.Ident("status.id"), minID)
}
if err := q.Scan(ctx, &childIDs); err != nil {
if err != sql.ErrNoRows {
log.Errorf(ctx, "error getting children for %q: %v", status.ID, err)
}
return return
} }
for _, id := range childIDs { for _, id := range childIDs {
if id <= minID {
continue
}
// Fetch child with ID from database // Fetch child with ID from database
child, err := s.GetStatusByID(ctx, id) child, err := s.GetStatusByID(ctx, id)
if err != nil { if err != nil {
@ -553,48 +560,80 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status,
} }
} }
func (s *statusDB) CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, error) { func (s *statusDB) GetStatusReplies(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
return s.db. statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
NewSelect(). if err != nil {
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")). return nil, err
Where("? = ?", bun.Ident("status.in_reply_to_id"), status.ID). }
Count(ctx) return s.GetStatusesByIDs(ctx, statusIDs)
} }
func (s *statusDB) CountStatusReblogs(ctx context.Context, status *gtsmodel.Status) (int, error) { func (s *statusDB) CountStatusReplies(ctx context.Context, statusID string) (int, error) {
return s.db. statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
NewSelect(). return len(statusIDs), err
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID).
Count(ctx)
} }
func (s *statusDB) CountStatusFaves(ctx context.Context, status *gtsmodel.Status) (int, error) { func (s *statusDB) getStatusReplyIDs(ctx context.Context, statusID string) ([]string, error) {
return s.db. return s.state.Caches.GTS.InReplyToIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status reply IDs not in cache, perform DB query!
if err := s.db.
NewSelect(). NewSelect().
TableExpr("? AS ?", bun.Ident("status_faves"), bun.Ident("status_fave")). Table("statuses").
Where("? = ?", bun.Ident("status_fave.status_id"), status.ID). Column("id").
Count(ctx) Where("? = ?", bun.Ident("in_reply_to_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, s.db.ProcessError(err)
} }
func (s *statusDB) IsStatusFavedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) { return statusIDs, nil
q := s.db. })
NewSelect().
TableExpr("? AS ?", bun.Ident("status_faves"), bun.Ident("status_fave")).
Where("? = ?", bun.Ident("status_fave.status_id"), status.ID).
Where("? = ?", bun.Ident("status_fave.account_id"), accountID)
return s.db.Exists(ctx, q)
} }
func (s *statusDB) IsStatusRebloggedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) { func (s *statusDB) GetStatusBoosts(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
q := s.db. statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
NewSelect(). if err != nil {
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")). return nil, err
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID). }
Where("? = ?", bun.Ident("status.account_id"), accountID) return s.GetStatusesByIDs(ctx, statusIDs)
}
return s.db.Exists(ctx, q) func (s *statusDB) IsStatusBoostedBy(ctx context.Context, statusID string, accountID string) (bool, error) {
boost, err := s.GetStatusBoost(
gtscontext.SetBarebones(ctx),
statusID,
accountID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (boost != nil), nil
}
func (s *statusDB) CountStatusBoosts(ctx context.Context, statusID string) (int, error) {
statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
return len(statusIDs), err
}
func (s *statusDB) getStatusBoostIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.BoostOfIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status boost IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("statuses").
Column("id").
Where("? = ?", bun.Ident("boost_of_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, s.db.ProcessError(err)
}
return statusIDs, nil
})
} }
func (s *statusDB) IsStatusMutedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) { func (s *statusDB) IsStatusMutedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) {
@ -616,16 +655,3 @@ func (s *statusDB) IsStatusBookmarkedBy(ctx context.Context, status *gtsmodel.St
return s.db.Exists(ctx, q) return s.db.Exists(ctx, q)
} }
func (s *statusDB) GetStatusReblogs(ctx context.Context, status *gtsmodel.Status) ([]*gtsmodel.Status, error) {
reblogs := []*gtsmodel.Status{}
q := s.
newStatusQ(&reblogs).
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID)
if err := q.Scan(ctx); err != nil {
return nil, s.db.ProcessError(err)
}
return reblogs, nil
}

View file

@ -19,6 +19,7 @@ package bundb
import ( import (
"context" "context"
"database/sql"
"errors" "errors"
"fmt" "fmt"
@ -44,8 +45,14 @@ func (s *statusFaveDB) GetStatusFave(ctx context.Context, accountID string, stat
return s.db. return s.db.
NewSelect(). NewSelect().
Model(fave). Model(fave).
Where("? = ?", bun.Ident("account_id"), accountID). Where("status_fave.account_id = ?", accountID).
Where("? = ?", bun.Ident("status_id"), statusID). Where("status_fave.status_id = ?", statusID).
// Our old code actually allowed a status to
// be faved multiple times by the same author,
// so limit our query + order to fetch latest.
Order("status_fave.id DESC"). // our IDs are timestamped
Limit(1).
Scan(ctx) Scan(ctx)
}, },
accountID, accountID,
@ -89,63 +96,68 @@ func (s *statusFaveDB) getStatusFave(ctx context.Context, lookup string, dbQuery
return fave, nil return fave, nil
} }
// Fetch the status fave author account. // Populate the status favourite model.
fave.Account, err = s.state.DB.GetAccountByID( if err := s.PopulateStatusFave(ctx, fave); err != nil {
gtscontext.SetBarebones(ctx), return nil, fmt.Errorf("error(s) populating status fave: %w", err)
fave.AccountID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave account %q: %w", fave.AccountID, err)
}
// Fetch the status fave target account.
fave.TargetAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
fave.TargetAccountID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave target account %q: %w", fave.TargetAccountID, err)
}
// Fetch the status fave target status.
fave.Status, err = s.state.DB.GetStatusByID(
gtscontext.SetBarebones(ctx),
fave.StatusID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave status %q: %w", fave.StatusID, err)
} }
return fave, nil return fave, nil
} }
func (s *statusFaveDB) GetStatusFavesForStatus(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error) { func (s *statusFaveDB) GetStatusFaves(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error) {
ids := []string{} // Fetch the status fave IDs for status.
faveIDs, err := s.getStatusFaveIDs(ctx, statusID)
if err := s.db. if err != nil {
NewSelect(). return nil, err
Table("status_faves").
Column("id").
Where("? = ?", bun.Ident("status_id"), statusID).
Scan(ctx, &ids); err != nil {
return nil, s.db.ProcessError(err)
} }
faves := make([]*gtsmodel.StatusFave, 0, len(ids)) // Preallocate a slice of expected status fave capacity.
faves := make([]*gtsmodel.StatusFave, 0, len(faveIDs))
for _, id := range ids { for _, id := range faveIDs {
// Fetch status fave model for each ID.
fave, err := s.GetStatusFaveByID(ctx, id) fave, err := s.GetStatusFaveByID(ctx, id)
if err != nil { if err != nil {
log.Errorf(ctx, "error getting status fave %q: %v", id, err) log.Errorf(ctx, "error getting status fave %q: %v", id, err)
continue continue
} }
faves = append(faves, fave) faves = append(faves, fave)
} }
return faves, nil return faves, nil
} }
func (s *statusFaveDB) IsStatusFavedBy(ctx context.Context, statusID string, accountID string) (bool, error) {
fave, err := s.GetStatusFave(ctx, accountID, statusID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (fave != nil), nil
}
func (s *statusFaveDB) CountStatusFaves(ctx context.Context, statusID string) (int, error) {
faveIDs, err := s.getStatusFaveIDs(ctx, statusID)
return len(faveIDs), err
}
func (s *statusFaveDB) getStatusFaveIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.StatusFaveIDs().Load(statusID, func() ([]string, error) {
var faveIDs []string
// Status fave IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("status_faves").
Column("id").
Where("? = ?", bun.Ident("status_id"), statusID).
Scan(ctx, &faveIDs); err != nil {
return nil, s.db.ProcessError(err)
}
return faveIDs, nil
})
}
func (s *statusFaveDB) PopulateStatusFave(ctx context.Context, statusFave *gtsmodel.StatusFave) error { func (s *statusFaveDB) PopulateStatusFave(ctx context.Context, statusFave *gtsmodel.StatusFave) error {
var ( var (
err error err error
@ -203,26 +215,32 @@ func (s *statusFaveDB) PutStatusFave(ctx context.Context, fave *gtsmodel.StatusF
} }
func (s *statusFaveDB) DeleteStatusFaveByID(ctx context.Context, id string) error { func (s *statusFaveDB) DeleteStatusFaveByID(ctx context.Context, id string) error {
defer s.state.Caches.GTS.StatusFave().Invalidate("ID", id) var statusID string
// Load fave into cache before attempting a delete, // Perform DELETE on status fave,
// as we need it cached in order to trigger the invalidate // returning the status ID it was for.
// callback. This in turn invalidates others. if _, err := s.db.NewDelete().
_, err := s.GetStatusFaveByID(gtscontext.SetBarebones(ctx), id) Table("status_faves").
if err != nil { Where("id = ?", id).
if errors.Is(err, db.ErrNoEntries) { Returning("status_id").
// not an issue. Exec(ctx, &statusID); err != nil {
if err == sql.ErrNoRows {
// Not an issue, only due
// to us doing a RETURNING.
err = nil err = nil
} }
return err return s.db.ProcessError(err)
} }
// Finally delete fave from DB. if statusID != "" {
_, err = s.db.NewDelete(). // Invalidate any cached status faves for this status.
Table("status_faves"). s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
Where("? = ?", bun.Ident("id"), id).
Exec(ctx) // Invalidate any cached status fave IDs for this status.
return s.db.ProcessError(err) s.state.Caches.GTS.StatusFaveIDs().Invalidate(statusID)
}
return nil
} }
func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) error { func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) error {
@ -230,12 +248,13 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
return errors.New("DeleteStatusFaves: one of targetAccountID or originAccountID must be set") return errors.New("DeleteStatusFaves: one of targetAccountID or originAccountID must be set")
} }
var faveIDs []string var statusIDs []string
q := s.db. // Prepare DELETE query returning
NewSelect(). // the deleted faves for status IDs.
Column("id"). q := s.db.NewDelete().
Table("status_faves") Table("status_faves").
Returning("status_id")
if targetAccountID != "" { if targetAccountID != "" {
q = q.Where("? = ?", bun.Ident("target_account_id"), targetAccountID) q = q.Where("? = ?", bun.Ident("target_account_id"), targetAccountID)
@ -245,69 +264,46 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
q = q.Where("? = ?", bun.Ident("account_id"), originAccountID) q = q.Where("? = ?", bun.Ident("account_id"), originAccountID)
} }
if _, err := q.Exec(ctx, &faveIDs); err != nil { // Execute query, store favourited status IDs.
if _, err := q.Exec(ctx, &statusIDs); err != nil {
if err == sql.ErrNoRows {
// Not an issue, only due
// to us doing a RETURNING.
err = nil
}
return s.db.ProcessError(err) return s.db.ProcessError(err)
} }
defer func() { // Collate (deduplicating) status IDs.
// Invalidate all IDs on return. statusIDs = collate(func(i int) string {
for _, id := range faveIDs { return statusIDs[i]
}, len(statusIDs))
for _, id := range statusIDs {
// Invalidate any cached status faves for this status.
s.state.Caches.GTS.StatusFave().Invalidate("ID", id) s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Load all faves into cache, this *really* isn't great // Invalidate any cached status fave IDs for this status.
// but it is the only way we can ensure we invalidate all s.state.Caches.GTS.StatusFaveIDs().Invalidate(id)
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
} }
// Finally delete all from DB. return nil
_, err := s.db.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.db.ProcessError(err)
} }
func (s *statusFaveDB) DeleteStatusFavesForStatus(ctx context.Context, statusID string) error { func (s *statusFaveDB) DeleteStatusFavesForStatus(ctx context.Context, statusID string) error {
// Capture fave IDs in a RETURNING statement. // Delete all status faves for status.
var faveIDs []string if _, err := s.db.NewDelete().
q := s.db.
NewSelect().
Column("id").
Table("status_faves"). Table("status_faves").
Where("? = ?", bun.Ident("status_id"), statusID) Where("status_id = ?", statusID).
if _, err := q.Exec(ctx, &faveIDs); err != nil { Exec(ctx); err != nil {
return s.db.ProcessError(err) return s.db.ProcessError(err)
} }
defer func() { // Invalidate any cached status faves for this status.
// Invalidate all IDs on return. s.state.Caches.GTS.StatusFave().Invalidate("ID", statusID)
for _, id := range faveIDs {
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Load all faves into cache, this *really* isn't great // Invalidate any cached status fave IDs for this status.
// but it is the only way we can ensure we invalidate all s.state.Caches.GTS.StatusFaveIDs().Invalidate(statusID)
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
// Finally delete all from DB. return nil
_, err := s.db.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.db.ProcessError(err)
} }

View file

@ -35,7 +35,7 @@ type StatusFaveTestSuite struct {
func (suite *StatusFaveTestSuite) TestGetStatusFaves() { func (suite *StatusFaveTestSuite) TestGetStatusFaves() {
testStatus := suite.testStatuses["admin_account_status_1"] testStatus := suite.testStatuses["admin_account_status_1"]
faves, err := suite.db.GetStatusFavesForStatus(context.Background(), testStatus.ID) faves, err := suite.db.GetStatusFaves(context.Background(), testStatus.ID)
if err != nil { if err != nil {
suite.FailNow(err.Error()) suite.FailNow(err.Error())
} }
@ -51,7 +51,7 @@ func (suite *StatusFaveTestSuite) TestGetStatusFaves() {
func (suite *StatusFaveTestSuite) TestGetStatusFavesNone() { func (suite *StatusFaveTestSuite) TestGetStatusFavesNone() {
testStatus := suite.testStatuses["admin_account_status_4"] testStatus := suite.testStatuses["admin_account_status_4"]
faves, err := suite.db.GetStatusFavesForStatus(context.Background(), testStatus.ID) faves, err := suite.db.GetStatusFaves(context.Background(), testStatus.ID)
if err != nil { if err != nil {
suite.FailNow(err.Error()) suite.FailNow(err.Error())
} }

View file

@ -41,10 +41,10 @@ type Media interface {
// DeleteAttachment deletes the attachment with given ID from the database. // DeleteAttachment deletes the attachment with given ID from the database.
DeleteAttachment(ctx context.Context, id string) error DeleteAttachment(ctx context.Context, id string) error
// GetAttachments ... // GetAttachments fetches media attachments up to a given max ID, and at most limit.
GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error)
// GetRemoteAttachments ... // GetRemoteAttachments fetches media attachments with a non-empty domain, up to a given max ID, and at most limit.
GetRemoteAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) GetRemoteAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error)
// GetCachedAttachmentsOlderThan gets limit n remote attachments (including avatars and headers) older than // GetCachedAttachmentsOlderThan gets limit n remote attachments (including avatars and headers) older than

View file

@ -34,6 +34,9 @@ type Status interface {
// GetStatusByURL returns one status from the database, with no rel fields populated, only their linking ID / URIs // GetStatusByURL returns one status from the database, with no rel fields populated, only their linking ID / URIs
GetStatusByURL(ctx context.Context, uri string) (*gtsmodel.Status, error) GetStatusByURL(ctx context.Context, uri string) (*gtsmodel.Status, error)
// GetStatusBoost fetches the status whose boost_of_id column refers to boostOfID, authored by given account ID.
GetStatusBoost(ctx context.Context, boostOfID string, byAccountID string) (*gtsmodel.Status, error)
// PopulateStatus ensures that all sub-models of a status are populated (e.g. mentions, attachments, etc). // PopulateStatus ensures that all sub-models of a status are populated (e.g. mentions, attachments, etc).
PopulateStatus(ctx context.Context, status *gtsmodel.Status) error PopulateStatus(ctx context.Context, status *gtsmodel.Status) error
@ -46,21 +49,27 @@ type Status interface {
// DeleteStatusByID deletes one status from the database. // DeleteStatusByID deletes one status from the database.
DeleteStatusByID(ctx context.Context, id string) error DeleteStatusByID(ctx context.Context, id string) error
// CountStatusReplies returns the amount of replies recorded for a status, or an error if something goes wrong
CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, error)
// CountStatusReblogs returns the amount of reblogs/boosts recorded for a status, or an error if something goes wrong
CountStatusReblogs(ctx context.Context, status *gtsmodel.Status) (int, error)
// CountStatusFaves returns the amount of faves/likes recorded for a status, or an error if something goes wrong
CountStatusFaves(ctx context.Context, status *gtsmodel.Status) (int, error)
// GetStatuses gets a slice of statuses corresponding to the given status IDs. // GetStatuses gets a slice of statuses corresponding to the given status IDs.
GetStatusesByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Status, error) GetStatusesByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Status, error)
// GetStatusesUsingEmoji fetches all status models using emoji with given ID stored in their 'emojis' column. // GetStatusesUsingEmoji fetches all status models using emoji with given ID stored in their 'emojis' column.
GetStatusesUsingEmoji(ctx context.Context, emojiID string) ([]*gtsmodel.Status, error) GetStatusesUsingEmoji(ctx context.Context, emojiID string) ([]*gtsmodel.Status, error)
// GetStatusReplies returns the *direct* (i.e. in_reply_to_id column) replies to this status ID.
GetStatusReplies(ctx context.Context, statusID string) ([]*gtsmodel.Status, error)
// CountStatusReplies returns the number of stored *direct* (i.e. in_reply_to_id column) replies to this status ID.
CountStatusReplies(ctx context.Context, statusID string) (int, error)
// GetStatusBoosts returns all statuses whose boost_of_id column refer to given status ID.
GetStatusBoosts(ctx context.Context, statusID string) ([]*gtsmodel.Status, error)
// CountStatusBoosts returns the number of stored boosts for status ID.
CountStatusBoosts(ctx context.Context, statusID string) (int, error)
// IsStatusBoostedBy checks whether the given status ID is boosted by account ID.
IsStatusBoostedBy(ctx context.Context, statusID string, accountID string) (bool, error)
// GetStatusParents gets the parent statuses of a given status. // GetStatusParents gets the parent statuses of a given status.
// //
// If onlyDirect is true, only the immediate parent will be returned. // If onlyDirect is true, only the immediate parent will be returned.
@ -71,19 +80,9 @@ type Status interface {
// If onlyDirect is true, only the immediate children will be returned. // If onlyDirect is true, only the immediate children will be returned.
GetStatusChildren(ctx context.Context, status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) GetStatusChildren(ctx context.Context, status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error)
// IsStatusFavedBy checks if a given status has been faved by a given account ID
IsStatusFavedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error)
// IsStatusRebloggedBy checks if a given status has been reblogged/boosted by a given account ID
IsStatusRebloggedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error)
// IsStatusMutedBy checks if a given status has been muted by a given account ID // IsStatusMutedBy checks if a given status has been muted by a given account ID
IsStatusMutedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) IsStatusMutedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error)
// IsStatusBookmarkedBy checks if a given status has been bookmarked by a given account ID // IsStatusBookmarkedBy checks if a given status has been bookmarked by a given account ID
IsStatusBookmarkedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) IsStatusBookmarkedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error)
// GetStatusReblogs returns a slice of statuses that are a boost/reblog of the given status.
// This slice will be unfiltered, not taking account of blocks and whatnot, so filter it before serving it back to a user.
GetStatusReblogs(ctx context.Context, status *gtsmodel.Status) ([]*gtsmodel.Status, error)
} }

View file

@ -24,16 +24,15 @@ import (
) )
type StatusFave interface { type StatusFave interface {
// GetStatusFaveByAccountID gets one status fave created by the given // GetStatusFaveByAccountID gets one status fave created by the given accountID, targeting the given statusID.
// accountID, targeting the given statusID.
GetStatusFave(ctx context.Context, accountID string, statusID string) (*gtsmodel.StatusFave, error) GetStatusFave(ctx context.Context, accountID string, statusID string) (*gtsmodel.StatusFave, error)
// GetStatusFave returns one status fave with the given id. // GetStatusFave returns one status fave with the given id.
GetStatusFaveByID(ctx context.Context, id string) (*gtsmodel.StatusFave, error) GetStatusFaveByID(ctx context.Context, id string) (*gtsmodel.StatusFave, error)
// GetStatusFaves returns a slice of faves/likes of the given status. // GetStatusFaves returns a slice of faves/likes of the status with given ID.
// This slice will be unfiltered, not taking account of blocks and whatnot, so filter it before serving it back to a user. // This slice will be unfiltered, not taking account of blocks and whatnot, so filter it before serving it back to a user.
GetStatusFavesForStatus(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error) GetStatusFaves(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error)
// PopulateStatusFave ensures that all sub-models of a fave are populated (account, status, etc). // PopulateStatusFave ensures that all sub-models of a fave are populated (account, status, etc).
PopulateStatusFave(ctx context.Context, statusFave *gtsmodel.StatusFave) error PopulateStatusFave(ctx context.Context, statusFave *gtsmodel.StatusFave) error
@ -59,8 +58,13 @@ type StatusFave interface {
// At least one parameter must not be an empty string. // At least one parameter must not be an empty string.
DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) error DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) error
// DeleteStatusFavesForStatus deletes all status faves that target the // DeleteStatusFavesForStatus deletes all status faves that target the given status ID.
// given status ID. This is useful when a status has been deleted, and you need // This is useful when a status has been deleted, and you need to clean up after it.
// to clean up after it.
DeleteStatusFavesForStatus(ctx context.Context, statusID string) error DeleteStatusFavesForStatus(ctx context.Context, statusID string) error
// CountStatusFaves returns the number of status favourites registered for status with ID.
CountStatusFaves(ctx context.Context, statusID string) (int, error)
// IsStatusFavedBy returns whether the status with ID has been favourited by account with ID.
IsStatusFavedBy(ctx context.Context, statusID string, accountID string) (bool, error)
} }

View file

@ -19,16 +19,13 @@ package gtserror
import ( import (
"errors" "errors"
"fmt"
) )
// MultiError allows encapsulating multiple // MultiError allows encapsulating multiple
// errors under a singular instance, which // errors under a singular instance, which
// is useful when you only want to log on // is useful when you only want to log on
// errors, not return early / bubble up. // errors, not return early / bubble up.
type MultiError struct { type MultiError []error
e []error
}
// NewMultiError returns a *MultiError with // NewMultiError returns a *MultiError with
// the capacity of its underlying error slice // the capacity of its underlying error slice
@ -40,15 +37,13 @@ type MultiError struct {
// //
// If you don't know in advance what the capacity // If you don't know in advance what the capacity
// must be, just use new(MultiError) instead. // must be, just use new(MultiError) instead.
func NewMultiError(capacity int) *MultiError { func NewMultiError(capacity int) MultiError {
return &MultiError{ return make([]error, 0, capacity)
e: make([]error, 0, capacity),
}
} }
// Append the given error to the MultiError. // Append the given error to the MultiError.
func (m *MultiError) Append(err error) { func (m *MultiError) Append(err error) {
m.e = append(m.e, err) (*m) = append((*m), err)
} }
// Append the given format string to the MultiError. // Append the given format string to the MultiError.
@ -56,12 +51,13 @@ func (m *MultiError) Append(err error) {
// It is valid to use %w in the format string // It is valid to use %w in the format string
// to wrap any other errors. // to wrap any other errors.
func (m *MultiError) Appendf(format string, args ...any) { func (m *MultiError) Appendf(format string, args ...any) {
m.e = append(m.e, fmt.Errorf(format, args...)) err := newfAt(3, format, args...)
(*m) = append((*m), err)
} }
// Combine the MultiError into a single error. // Combine the MultiError into a single error.
// //
// Unwrap will work on the returned error as expected. // Unwrap will work on the returned error as expected.
func (m MultiError) Combine() error { func (m MultiError) Combine() error {
return errors.Join(m.e...) return errors.Join(m...)
} }

View file

@ -15,22 +15,22 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package gtserror package gtserror_test
import ( import (
"errors" "errors"
"testing" "testing"
"github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
) )
func TestMultiError(t *testing.T) { func TestMultiError(t *testing.T) {
errs := MultiError{ errs := gtserror.MultiError([]error{
e: []error{
db.ErrNoEntries, db.ErrNoEntries,
errors.New("oopsie woopsie we did a fucky wucky etc"), errors.New("oopsie woopsie we did a fucky wucky etc"),
}, })
}
errs.Appendf("appended + wrapped error: %w", db.ErrAlreadyExists) errs.Appendf("appended + wrapped error: %w", db.ErrAlreadyExists)
err := errs.Combine() err := errs.Combine()
@ -50,14 +50,14 @@ func TestMultiError(t *testing.T) {
errString := err.Error() errString := err.Error()
expected := `sql: no rows in result set expected := `sql: no rows in result set
oopsie woopsie we did a fucky wucky etc oopsie woopsie we did a fucky wucky etc
appended + wrapped error: already exists` TestMultiError: appended + wrapped error: already exists`
if errString != expected { if errString != expected {
t.Errorf("errString '%s' should be '%s'", errString, expected) t.Errorf("errString '%s' should be '%s'", errString, expected)
} }
} }
func TestMultiErrorEmpty(t *testing.T) { func TestMultiErrorEmpty(t *testing.T) {
err := new(MultiError).Combine() err := new(gtserror.MultiError).Combine()
if err != nil { if err != nil {
t.Errorf("should be nil") t.Errorf("should be nil")
} }

View file

@ -330,7 +330,7 @@ statusLoop:
}) })
// Look for any boosts of this status in DB. // Look for any boosts of this status in DB.
boosts, err := p.state.DB.GetStatusReblogs(ctx, status) boosts, err := p.state.DB.GetStatusBoosts(ctx, status.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) { if err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("error fetching status reblogs for %s: %w", status.ID, err) return gtserror.Newf("error fetching status reblogs for %s: %w", status.ID, err)
} }

View file

@ -380,6 +380,8 @@ func (p *Processor) notify(
// wipeStatus contains common logic used to totally delete a status // wipeStatus contains common logic used to totally delete a status
// + all its attachments, notifications, boosts, and timeline entries. // + all its attachments, notifications, boosts, and timeline entries.
func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Status, deleteAttachments bool) error { func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Status, deleteAttachments bool) error {
var errs gtserror.MultiError
// either delete all attachments for this status, or simply // either delete all attachments for this status, or simply
// unattach all attachments for this status, so they'll be // unattach all attachments for this status, so they'll be
// cleaned later by a separate process; reason to unattach rather // cleaned later by a separate process; reason to unattach rather
@ -389,14 +391,14 @@ func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Sta
// todo: p.state.DB.DeleteAttachmentsForStatus // todo: p.state.DB.DeleteAttachmentsForStatus
for _, a := range statusToDelete.AttachmentIDs { for _, a := range statusToDelete.AttachmentIDs {
if err := p.media.Delete(ctx, a); err != nil { if err := p.media.Delete(ctx, a); err != nil {
return err errs.Appendf("error deleting media: %w", err)
} }
} }
} else { } else {
// todo: p.state.DB.UnattachAttachmentsForStatus // todo: p.state.DB.UnattachAttachmentsForStatus
for _, a := range statusToDelete.AttachmentIDs { for _, a := range statusToDelete.AttachmentIDs {
if _, err := p.media.Unattach(ctx, statusToDelete.Account, a); err != nil { if _, err := p.media.Unattach(ctx, statusToDelete.Account, a); err != nil {
return err errs.Appendf("error unattaching media: %w", err)
} }
} }
} }
@ -405,44 +407,55 @@ func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Sta
// todo: p.state.DB.DeleteMentionsForStatus // todo: p.state.DB.DeleteMentionsForStatus
for _, id := range statusToDelete.MentionIDs { for _, id := range statusToDelete.MentionIDs {
if err := p.state.DB.DeleteMentionByID(ctx, id); err != nil { if err := p.state.DB.DeleteMentionByID(ctx, id); err != nil {
return err errs.Appendf("error deleting status mention: %w", err)
} }
} }
// delete all notification entries generated by this status // delete all notification entries generated by this status
if err := p.state.DB.DeleteNotificationsForStatus(ctx, statusToDelete.ID); err != nil { if err := p.state.DB.DeleteNotificationsForStatus(ctx, statusToDelete.ID); err != nil {
return err errs.Appendf("error deleting status notifications: %w", err)
} }
// delete all bookmarks that point to this status // delete all bookmarks that point to this status
if err := p.state.DB.DeleteStatusBookmarksForStatus(ctx, statusToDelete.ID); err != nil { if err := p.state.DB.DeleteStatusBookmarksForStatus(ctx, statusToDelete.ID); err != nil {
return err errs.Appendf("error deleting status bookmarks: %w", err)
} }
// delete all faves of this status // delete all faves of this status
if err := p.state.DB.DeleteStatusFavesForStatus(ctx, statusToDelete.ID); err != nil { if err := p.state.DB.DeleteStatusFavesForStatus(ctx, statusToDelete.ID); err != nil {
return err errs.Appendf("error deleting status faves: %w", err)
} }
// delete all boosts for this status + remove them from timelines // delete all boosts for this status + remove them from timelines
if boosts, err := p.state.DB.GetStatusReblogs(ctx, statusToDelete); err == nil { boosts, err := p.state.DB.GetStatusBoosts(
// we MUST set a barebones context here,
// as depending on where it came from the
// original BoostOf may already be gone.
gtscontext.SetBarebones(ctx),
statusToDelete.ID)
if err != nil {
errs.Appendf("error fetching status boosts: %w", err)
}
for _, b := range boosts { for _, b := range boosts {
if err := p.deleteStatusFromTimelines(ctx, b.ID); err != nil { if err := p.deleteStatusFromTimelines(ctx, b.ID); err != nil {
return err errs.Appendf("error deleting boost from timelines: %w", err)
} }
if err := p.state.DB.DeleteStatusByID(ctx, b.ID); err != nil { if err := p.state.DB.DeleteStatusByID(ctx, b.ID); err != nil {
return err errs.Appendf("error deleting boost: %w", err)
}
} }
} }
// delete this status from any and all timelines // delete this status from any and all timelines
if err := p.deleteStatusFromTimelines(ctx, statusToDelete.ID); err != nil { if err := p.deleteStatusFromTimelines(ctx, statusToDelete.ID); err != nil {
return err errs.Appendf("error deleting status from timelines: %w", err)
} }
// delete the status itself // finally, delete the status itself
return p.state.DB.DeleteStatusByID(ctx, statusToDelete.ID) if err := p.state.DB.DeleteStatusByID(ctx, statusToDelete.ID); err != nil {
errs.Appendf("error deleting status: %w", err)
}
return errs.Combine()
} }
// deleteStatusFromTimelines completely removes the given status from all timelines. // deleteStatusFromTimelines completely removes the given status from all timelines.

View file

@ -106,47 +106,24 @@ func (p *Processor) BoostRemove(ctx context.Context, requestingAccount *gtsmodel
return nil, gtserror.NewErrorNotFound(errors.New("status is not visible")) return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
} }
// check if we actually have a boost for this status // Check whether the requesting account has boosted the given status ID.
var toUnboost bool boost, err := p.state.DB.GetStatusBoost(ctx, targetStatusID, requestingAccount.ID)
gtsBoost := &gtsmodel.Status{}
where := []db.Where{
{
Key: "boost_of_id",
Value: targetStatusID,
},
{
Key: "account_id",
Value: requestingAccount.ID,
},
}
err = p.state.DB.GetWhere(ctx, where, gtsBoost)
if err == nil {
// we have a boost
toUnboost = true
}
if err != nil { if err != nil {
// something went wrong in the db finding the boost return nil, gtserror.NewErrorNotFound(fmt.Errorf("error checking status boost %s: %w", targetStatusID, err))
if err != db.ErrNoEntries {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error fetching existing boost from database: %s", err))
}
// we just don't have a boost
toUnboost = false
} }
if toUnboost { if boost != nil {
// pin some stuff onto the boost while we have it out of the db // pin some stuff onto the boost while we have it out of the db
gtsBoost.Account = requestingAccount boost.Account = requestingAccount
gtsBoost.BoostOf = targetStatus boost.BoostOf = targetStatus
gtsBoost.BoostOfAccount = targetStatus.Account boost.BoostOfAccount = targetStatus.Account
gtsBoost.BoostOf.Account = targetStatus.Account boost.BoostOf.Account = targetStatus.Account
// send it back to the processor for async processing // send it back to the processor for async processing
p.state.Workers.EnqueueClientAPI(ctx, messages.FromClientAPI{ p.state.Workers.EnqueueClientAPI(ctx, messages.FromClientAPI{
APObjectType: ap.ActivityAnnounce, APObjectType: ap.ActivityAnnounce,
APActivityType: ap.ActivityUndo, APActivityType: ap.ActivityUndo,
GTSModel: gtsBoost, GTSModel: boost,
OriginAccount: requestingAccount, OriginAccount: requestingAccount,
TargetAccount: targetStatus.Account, TargetAccount: targetStatus.Account,
}) })
@ -189,15 +166,15 @@ func (p *Processor) StatusBoostedBy(ctx context.Context, requestingAccount *gtsm
return nil, gtserror.NewErrorNotFound(err) return nil, gtserror.NewErrorNotFound(err)
} }
statusReblogs, err := p.state.DB.GetStatusReblogs(ctx, targetStatus) statusBoosts, err := p.state.DB.GetStatusBoosts(ctx, targetStatus.ID)
if err != nil { if err != nil {
err = fmt.Errorf("BoostedBy: error seeing who boosted status: %s", err) err = fmt.Errorf("BoostedBy: error seeing who boosted status: %s", err)
return nil, gtserror.NewErrorNotFound(err) return nil, gtserror.NewErrorNotFound(err)
} }
// filter account IDs so the user doesn't see accounts they blocked or which blocked them // filter account IDs so the user doesn't see accounts they blocked or which blocked them
accountIDs := make([]string, 0, len(statusReblogs)) accountIDs := make([]string, 0, len(statusBoosts))
for _, s := range statusReblogs { for _, s := range statusBoosts {
blocked, err := p.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, s.AccountID) blocked, err := p.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, s.AccountID)
if err != nil { if err != nil {
err = fmt.Errorf("BoostedBy: error checking blocks: %s", err) err = fmt.Errorf("BoostedBy: error checking blocks: %s", err)

View file

@ -112,7 +112,7 @@ func (p *Processor) FavedBy(ctx context.Context, requestingAccount *gtsmodel.Acc
return nil, errWithCode return nil, errWithCode
} }
statusFaves, err := p.state.DB.GetStatusFavesForStatus(ctx, targetStatus.ID) statusFaves, err := p.state.DB.GetStatusFaves(ctx, targetStatus.ID)
if err != nil { if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("FavedBy: error seeing who faved status: %s", err)) return nil, gtserror.NewErrorNotFound(fmt.Errorf("FavedBy: error seeing who faved status: %s", err))
} }

View file

@ -600,17 +600,17 @@ func (c *converter) StatusToAPIStatus(ctx context.Context, s *gtsmodel.Status, r
return nil, fmt.Errorf("error converting status author: %w", err) return nil, fmt.Errorf("error converting status author: %w", err)
} }
repliesCount, err := c.db.CountStatusReplies(ctx, s) repliesCount, err := c.db.CountStatusReplies(ctx, s.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error counting replies: %w", err) return nil, fmt.Errorf("error counting replies: %w", err)
} }
reblogsCount, err := c.db.CountStatusReblogs(ctx, s) reblogsCount, err := c.db.CountStatusBoosts(ctx, s.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error counting reblogs: %w", err) return nil, fmt.Errorf("error counting reblogs: %w", err)
} }
favesCount, err := c.db.CountStatusFaves(ctx, s) favesCount, err := c.db.CountStatusFaves(ctx, s.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error counting faves: %w", err) return nil, fmt.Errorf("error counting faves: %w", err)
} }

View file

@ -40,13 +40,13 @@ func (c *converter) interactionsWithStatusForAccount(ctx context.Context, s *gts
si := &statusInteractions{} si := &statusInteractions{}
if requestingAccount != nil { if requestingAccount != nil {
faved, err := c.db.IsStatusFavedBy(ctx, s, requestingAccount.ID) faved, err := c.db.IsStatusFavedBy(ctx, s.ID, requestingAccount.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error checking if requesting account has faved status: %s", err) return nil, fmt.Errorf("error checking if requesting account has faved status: %s", err)
} }
si.Faved = faved si.Faved = faved
reblogged, err := c.db.IsStatusRebloggedBy(ctx, s, requestingAccount.ID) reblogged, err := c.db.IsStatusBoostedBy(ctx, s.ID, requestingAccount.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error checking if requesting account has reblogged status: %s", err) return nil, fmt.Errorf("error checking if requesting account has reblogged status: %s", err)
} }

View file

@ -21,12 +21,14 @@ EXPECT=$(cat << "EOF"
"account-mem-ratio": 18, "account-mem-ratio": 18,
"account-note-mem-ratio": 0.1, "account-note-mem-ratio": 0.1,
"block-mem-ratio": 3, "block-mem-ratio": 3,
"boost-of-ids-mem-ratio": 3,
"emoji-category-mem-ratio": 0.1, "emoji-category-mem-ratio": 0.1,
"emoji-mem-ratio": 3, "emoji-mem-ratio": 3,
"follow-ids-mem-ratio": 4, "follow-ids-mem-ratio": 4,
"follow-mem-ratio": 4, "follow-mem-ratio": 4,
"follow-request-ids-mem-ratio": 2, "follow-request-ids-mem-ratio": 2,
"follow-request-mem-ratio": 2, "follow-request-mem-ratio": 2,
"in-reply-to-ids-mem-ratio": 3,
"instance-mem-ratio": 1, "instance-mem-ratio": 1,
"list-entry-mem-ratio": 3, "list-entry-mem-ratio": 3,
"list-mem-ratio": 3, "list-mem-ratio": 3,
@ -36,6 +38,7 @@ EXPECT=$(cat << "EOF"
"mention-mem-ratio": 5, "mention-mem-ratio": 5,
"notification-mem-ratio": 5, "notification-mem-ratio": 5,
"report-mem-ratio": 1, "report-mem-ratio": 1,
"status-fave-ids-mem-ratio": 3,
"status-fave-mem-ratio": 5, "status-fave-mem-ratio": 5,
"status-mem-ratio": 18, "status-mem-ratio": 18,
"tag-mem-ratio": 3, "tag-mem-ratio": 3,

View file

@ -11,28 +11,7 @@ import (
"codeberg.org/gruf/go-errors/v2" "codeberg.org/gruf/go-errors/v2"
) )
type result struct { var ErrUnsupportedZero = errors.New("")
// Result primary key
PKey int64
// keys accessible under
Keys cacheKeys
// cached value
Value any
// cached error
Error error
}
// getResultValue is a safe way of casting and fetching result value.
func getResultValue[T any](res *result) T {
v, ok := res.Value.(T)
if !ok {
fmt.Fprintf(os.Stderr, "!! BUG: unexpected value type in result: %T\n", res.Value)
}
return v
}
// Lookup represents a struct object lookup method in the cache. // Lookup represents a struct object lookup method in the cache.
type Lookup struct { type Lookup struct {
@ -255,13 +234,15 @@ func (c *Cache[T]) Load(lookup string, load func() (T, error), keyParts ...any)
evict = c.store(res) evict = c.store(res)
} }
// Catch and return error // Catch and return cached error
if res.Error != nil { if err := res.Error; err != nil {
return zero, res.Error return zero, err
} }
// Return a copy of value from cache // Copy value from cached result.
return c.copy(getResultValue[T](res)), nil v := c.copy(getResultValue[T](res))
return v, nil
} }
// Store will call the given store function, and on success store the value in the cache as a positive result. // Store will call the given store function, and on success store the value in the cache as a positive result.
@ -332,11 +313,13 @@ func (c *Cache[T]) Has(lookup string, keyParts ...any) bool {
} }
} }
// Check for result AND non-error result.
ok := (res != nil && res.Error == nil)
// Done with lock // Done with lock
c.cache.Unlock() c.cache.Unlock()
// Check for result AND non-error result. return ok
return (res != nil && res.Error == nil)
} }
// Invalidate will invalidate any result from the cache found under given lookup and key parts. // Invalidate will invalidate any result from the cache found under given lookup and key parts.
@ -407,13 +390,18 @@ func (c *Cache[T]) store(res *result) (evict func()) {
key.info.pkeys[key.key] = pkeys key.info.pkeys[key.key] = pkeys
} }
// Acquire new cache entry.
entry := simple.GetEntry()
entry.Key = res.PKey
entry.Value = res
evictFn := func(_ int64, entry *simple.Entry) {
// on evict during set, store evicted result.
toEvict = append(toEvict, entry.Value.(*result))
}
// Store main entry under primary key, catch evicted. // Store main entry under primary key, catch evicted.
c.cache.Cache.SetWithHook(res.PKey, &simple.Entry{ c.cache.Cache.SetWithHook(res.PKey, entry, evictFn)
Key: res.PKey,
Value: res,
}, func(_ int64, item *simple.Entry) {
toEvict = append(toEvict, item.Value.(*result))
})
if len(toEvict) == 0 { if len(toEvict) == 0 {
// none evicted. // none evicted.
@ -421,9 +409,35 @@ func (c *Cache[T]) store(res *result) (evict func()) {
} }
return func() { return func() {
for _, res := range toEvict { for i := range toEvict {
// Rescope result.
res := toEvict[i]
// Call evict hook on each entry. // Call evict hook on each entry.
c.cache.Evict(res.PKey, res) c.cache.Evict(res.PKey, res)
} }
} }
} }
type result struct {
// Result primary key
PKey int64
// keys accessible under
Keys cacheKeys
// cached value
Value any
// cached error
Error error
}
// getResultValue is a safe way of casting and fetching result value.
func getResultValue[T any](res *result) T {
v, ok := res.Value.(T)
if !ok {
fmt.Fprintf(os.Stderr, "!! BUG: unexpected value type in result: %T\n", res.Value)
}
return v
}

View file

@ -47,27 +47,32 @@ func (sk structKeys) generate(a any) []cacheKey {
buf := getBuf() buf := getBuf()
defer putBuf(buf) defer putBuf(buf)
outer:
for i := range sk { for i := range sk {
// Reset buffer // Reset buffer
buf.B = buf.B[:0] buf.Reset()
// Append each field value to buffer. // Append each field value to buffer.
for _, field := range sk[i].fields { for _, field := range sk[i].fields {
fv := v.Field(field.index) fv := v.Field(field.index)
fi := fv.Interface() fi := fv.Interface()
buf.B = field.mangle(buf.B, fi)
// Mangle this key part into buffer.
ok := field.manglePart(buf, fi)
if !ok {
// don't generate keys
// for zero value parts.
continue outer
}
// Append part separator.
buf.B = append(buf.B, '.') buf.B = append(buf.B, '.')
} }
// Drop last '.' // Drop last '.'
buf.Truncate(1) buf.Truncate(1)
// Don't generate keys for zero values
if allowZero := sk[i].zero == ""; // nocollapse
!allowZero && buf.String() == sk[i].zero {
continue
}
// Append new cached key to slice // Append new cached key to slice
keys = append(keys, cacheKey{ keys = append(keys, cacheKey{
info: &sk[i], info: &sk[i],
@ -114,14 +119,6 @@ type structKey struct {
// period ('.') separated struct field names. // period ('.') separated struct field names.
name string name string
// zero is the possible zero value for this key.
// if set, this will _always_ be non-empty, as
// the mangled cache key will never be empty.
//
// i.e. zero = "" --> allow zero value keys
// zero != "" --> don't allow zero value keys
zero string
// unique determines whether this structKey supports // unique determines whether this structKey supports
// multiple or just the singular unique result. // multiple or just the singular unique result.
unique bool unique bool
@ -135,47 +132,10 @@ type structKey struct {
pkeys map[string][]int64 pkeys map[string][]int64
} }
type structField struct {
// index is the reflect index of this struct field.
index int
// mangle is the mangler function for
// serializing values of this struct field.
mangle mangler.Mangler
}
// genKey generates a cache key string for given key parts (i.e. serializes them using "go-mangler").
func (sk *structKey) genKey(parts []any) string {
// Check this expected no. key parts.
if len(parts) != len(sk.fields) {
panic(fmt.Sprintf("incorrect no. key parts provided: want=%d received=%d", len(parts), len(sk.fields)))
}
// Acquire byte buffer
buf := getBuf()
defer putBuf(buf)
buf.Reset()
// Encode each key part
for i, part := range parts {
buf.B = sk.fields[i].mangle(buf.B, part)
buf.B = append(buf.B, '.')
}
// Drop last '.'
buf.Truncate(1)
// Return string copy
return string(buf.B)
}
// newStructKey will generate a structKey{} information object for user-given lookup // newStructKey will generate a structKey{} information object for user-given lookup
// key information, and the receiving generic paramter's type information. Panics on error. // key information, and the receiving generic paramter's type information. Panics on error.
func newStructKey(lk Lookup, t reflect.Type) structKey { func newStructKey(lk Lookup, t reflect.Type) structKey {
var ( var sk structKey
sk structKey
zeros []any
)
// Set the lookup name // Set the lookup name
sk.name = lk.Name sk.name = lk.Name
@ -183,9 +143,6 @@ func newStructKey(lk Lookup, t reflect.Type) structKey {
// Split dot-separated lookup to get // Split dot-separated lookup to get
// the individual struct field names // the individual struct field names
names := strings.Split(lk.Name, ".") names := strings.Split(lk.Name, ".")
if len(names) == 0 {
panic("no key fields specified")
}
// Allocate the mangler and field indices slice. // Allocate the mangler and field indices slice.
sk.fields = make([]structField, len(names)) sk.fields = make([]structField, len(names))
@ -213,16 +170,12 @@ func newStructKey(lk Lookup, t reflect.Type) structKey {
sk.fields[i].mangle = mangler.Get(ft.Type) sk.fields[i].mangle = mangler.Get(ft.Type)
if !lk.AllowZero { if !lk.AllowZero {
// Append the zero value interface // Append the mangled zero value interface
zeros = append(zeros, v.Interface()) zero := sk.fields[i].mangle(nil, v.Interface())
sk.fields[i].zero = string(zero)
} }
} }
if len(zeros) > 0 {
// Generate zero value string
sk.zero = sk.genKey(zeros)
}
// Set unique lookup flag. // Set unique lookup flag.
sk.unique = !lk.Multi sk.unique = !lk.Multi
@ -232,6 +185,68 @@ func newStructKey(lk Lookup, t reflect.Type) structKey {
return sk return sk
} }
// genKey generates a cache key string for given key parts (i.e. serializes them using "go-mangler").
func (sk *structKey) genKey(parts []any) string {
// Check this expected no. key parts.
if len(parts) != len(sk.fields) {
panic(fmt.Sprintf("incorrect no. key parts provided: want=%d received=%d", len(parts), len(sk.fields)))
}
// Acquire byte buffer
buf := getBuf()
defer putBuf(buf)
buf.Reset()
for i, part := range parts {
// Mangle this key part into buffer.
// specifically ignoring whether this
// is returning a zero value key part.
_ = sk.fields[i].manglePart(buf, part)
// Append part separator.
buf.B = append(buf.B, '.')
}
// Drop last '.'
buf.Truncate(1)
// Return string copy
return string(buf.B)
}
type structField struct {
// index is the reflect index of this struct field.
index int
// zero is the possible zero value for this
// key part. if set, this will _always_ be
// non-empty due to how the mangler works.
//
// i.e. zero = "" --> allow zero value keys
// zero != "" --> don't allow zero value keys
zero string
// mangle is the mangler function for
// serializing values of this struct field.
mangle mangler.Mangler
}
// manglePart ...
func (field *structField) manglePart(buf *byteutil.Buffer, part any) bool {
// Start of part bytes.
start := len(buf.B)
// Mangle this key part into buffer.
buf.B = field.mangle(buf.B, part)
// End of part bytes.
end := len(buf.B)
// Return whether this is zero value.
return (field.zero == "" ||
string(buf.B[start:end]) != field.zero)
}
// isExported checks whether function name is exported. // isExported checks whether function name is exported.
func isExported(fnName string) bool { func isExported(fnName string) bool {
r, _ := utf8.DecodeRuneInString(fnName) r, _ := utf8.DecodeRuneInString(fnName)
@ -246,12 +261,12 @@ var bufPool = sync.Pool{
}, },
} }
// getBuf ... // getBuf acquires a byte buffer from memory pool.
func getBuf() *byteutil.Buffer { func getBuf() *byteutil.Buffer {
return bufPool.Get().(*byteutil.Buffer) return bufPool.Get().(*byteutil.Buffer)
} }
// putBuf ... // putBuf replaces a byte buffer back in memory pool.
func putBuf(buf *byteutil.Buffer) { func putBuf(buf *byteutil.Buffer) {
if buf.Cap() > int(^uint16(0)) { if buf.Cap() > int(^uint16(0)) {
return // drop large bufs return // drop large bufs

View file

@ -102,7 +102,7 @@ func (c *Cache[K, V]) Add(key K, value V) bool {
} }
// Alloc new entry. // Alloc new entry.
new := getEntry() new := GetEntry()
new.Key = key new.Key = key
new.Value = value new.Value = value
@ -111,7 +111,7 @@ func (c *Cache[K, V]) Add(key K, value V) bool {
evcK = item.Key.(K) evcK = item.Key.(K)
evcV = item.Value.(V) evcV = item.Value.(V)
ev = true ev = true
putEntry(item) PutEntry(item)
}) })
// Set hook func ptr. // Set hook func ptr.
@ -161,7 +161,7 @@ func (c *Cache[K, V]) Set(key K, value V) {
item.Value = value item.Value = value
} else { } else {
// Alloc new entry. // Alloc new entry.
new := getEntry() new := GetEntry()
new.Key = key new.Key = key
new.Value = value new.Value = value
@ -170,7 +170,7 @@ func (c *Cache[K, V]) Set(key K, value V) {
evcK = item.Key.(K) evcK = item.Key.(K)
evcV = item.Value.(V) evcV = item.Value.(V)
ev = true ev = true
putEntry(item) PutEntry(item)
}) })
} }
@ -311,7 +311,7 @@ func (c *Cache[K, V]) Invalidate(key K) (ok bool) {
_ = c.Cache.Delete(key) _ = c.Cache.Delete(key)
// Free entry // Free entry
putEntry(item) PutEntry(item)
// Set hook func ptrs. // Set hook func ptrs.
invalid = c.Invalid invalid = c.Invalid
@ -367,7 +367,7 @@ func (c *Cache[K, V]) InvalidateAll(keys ...K) (ok bool) {
invalid(k, v) invalid(k, v)
// Free this entry. // Free this entry.
putEntry(items[x]) PutEntry(items[x])
} }
} }
@ -410,7 +410,7 @@ func (c *Cache[K, V]) Trim(perc float64) {
invalid(k, v) invalid(k, v)
// Free this entry. // Free this entry.
putEntry(items[x]) PutEntry(items[x])
} }
} }
} }
@ -438,7 +438,7 @@ func (c *Cache[K, V]) locked(fn func()) {
func (c *Cache[K, V]) truncate(sz int, hook func(K, V)) []*Entry { func (c *Cache[K, V]) truncate(sz int, hook func(K, V)) []*Entry {
if hook == nil { if hook == nil {
// No hook to execute, simply release all truncated entries. // No hook to execute, simply release all truncated entries.
c.Cache.Truncate(sz, func(_ K, item *Entry) { putEntry(item) }) c.Cache.Truncate(sz, func(_ K, item *Entry) { PutEntry(item) })
return nil return nil
} }

View file

@ -6,8 +6,8 @@ import "sync"
// objects, regardless of cache type. // objects, regardless of cache type.
var entryPool sync.Pool var entryPool sync.Pool
// getEntry fetches an Entry from pool, or allocates new. // GetEntry fetches an Entry from pool, or allocates new.
func getEntry() *Entry { func GetEntry() *Entry {
v := entryPool.Get() v := entryPool.Get()
if v == nil { if v == nil {
return new(Entry) return new(Entry)
@ -15,8 +15,8 @@ func getEntry() *Entry {
return v.(*Entry) return v.(*Entry)
} }
// putEntry replaces an Entry in the pool. // PutEntry replaces an Entry in the pool.
func putEntry(e *Entry) { func PutEntry(e *Entry) {
e.Key = nil e.Key = nil
e.Value = nil e.Value = nil
entryPool.Put(e) entryPool.Put(e)

2
vendor/modules.txt vendored
View file

@ -13,7 +13,7 @@ codeberg.org/gruf/go-bytesize
# codeberg.org/gruf/go-byteutil v1.1.2 # codeberg.org/gruf/go-byteutil v1.1.2
## explicit; go 1.16 ## explicit; go 1.16
codeberg.org/gruf/go-byteutil codeberg.org/gruf/go-byteutil
# codeberg.org/gruf/go-cache/v3 v3.5.3 # codeberg.org/gruf/go-cache/v3 v3.5.5
## explicit; go 1.19 ## explicit; go 1.19
codeberg.org/gruf/go-cache/v3 codeberg.org/gruf/go-cache/v3
codeberg.org/gruf/go-cache/v3/result codeberg.org/gruf/go-cache/v3/result