gotosocial/internal/db/bundb/status.go

648 lines
18 KiB
Go
Raw Normal View History

// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bundb
import (
"context"
"errors"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
[chore] use our own logging implementation (#716) * first commit Signed-off-by: kim <grufwub@gmail.com> * replace logging with our own log library Signed-off-by: kim <grufwub@gmail.com> * fix imports Signed-off-by: kim <grufwub@gmail.com> * fix log imports Signed-off-by: kim <grufwub@gmail.com> * add license text Signed-off-by: kim <grufwub@gmail.com> * fix package import cycle between config and log package Signed-off-by: kim <grufwub@gmail.com> * fix empty kv.Fields{} being passed to WithFields() Signed-off-by: kim <grufwub@gmail.com> * fix uses of log.WithFields() with whitespace issues and empty slices Signed-off-by: kim <grufwub@gmail.com> * *linter related grumbling* Signed-off-by: kim <grufwub@gmail.com> * gofmt the codebase! also fix more log.WithFields() formatting issues Signed-off-by: kim <grufwub@gmail.com> * update testrig code to match new changes Signed-off-by: kim <grufwub@gmail.com> * fix error wrapping in non fmt.Errorf function Signed-off-by: kim <grufwub@gmail.com> * add benchmarking of log.Caller() vs non-cached Signed-off-by: kim <grufwub@gmail.com> * fix syslog tests, add standard build tags to test runner to ensure consistency Signed-off-by: kim <grufwub@gmail.com> * make syslog tests more robust Signed-off-by: kim <grufwub@gmail.com> * fix caller depth arithmatic (is that how you spell it?) Signed-off-by: kim <grufwub@gmail.com> * update to use unkeyed fields in kv.Field{} instances Signed-off-by: kim <grufwub@gmail.com> * update go-kv library Signed-off-by: kim <grufwub@gmail.com> * update libraries list Signed-off-by: kim <grufwub@gmail.com> * fuck you linter get nerfed Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
2022-07-19 08:47:55 +00:00
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/uptrace/bun"
)
type statusDB struct {
db *DB
state *state.State
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) GetStatusByID(ctx context.Context, id string) (*gtsmodel.Status, error) {
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
return s.getStatus(
ctx,
"ID",
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
func(status *gtsmodel.Status) error {
return s.db.NewSelect().Model(status).Where("? = ?", bun.Ident("status.id"), id).Scan(ctx)
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
},
id,
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
)
}
func (s *statusDB) GetStatusesByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Status, error) {
statuses := make([]*gtsmodel.Status, 0, len(ids))
for _, id := range ids {
// Attempt to fetch status from DB.
status, err := s.GetStatusByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting status %q: %v", id, err)
continue
}
// Append status to return slice.
statuses = append(statuses, status)
}
return statuses, nil
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) GetStatusByURI(ctx context.Context, uri string) (*gtsmodel.Status, error) {
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
return s.getStatus(
ctx,
"URI",
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
func(status *gtsmodel.Status) error {
return s.db.NewSelect().Model(status).Where("? = ?", bun.Ident("status.uri"), uri).Scan(ctx)
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
},
uri,
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
)
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) GetStatusByURL(ctx context.Context, url string) (*gtsmodel.Status, error) {
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
return s.getStatus(
ctx,
"URL",
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
func(status *gtsmodel.Status) error {
return s.db.NewSelect().Model(status).Where("? = ?", bun.Ident("status.url"), url).Scan(ctx)
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
},
url,
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
)
}
func (s *statusDB) GetStatusBoost(ctx context.Context, boostOfID string, byAccountID string) (*gtsmodel.Status, error) {
return s.getStatus(
ctx,
"BoostOfID.AccountID",
func(status *gtsmodel.Status) error {
return s.db.NewSelect().Model(status).
Where("status.boost_of_id = ?", boostOfID).
Where("status.account_id = ?", byAccountID).
// Our old code actually allowed a status to
// be boosted multiple times by the same author,
// so limit our query + order to fetch latest.
Order("status.id DESC"). // our IDs are timestamped
Limit(1).
Scan(ctx)
},
boostOfID, byAccountID,
)
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Status) error, keyParts ...any) (*gtsmodel.Status, error) {
// Fetch status from database cache with loader callback
status, err := s.state.Caches.GTS.Status().Load(lookup, func() (*gtsmodel.Status, error) {
var status gtsmodel.Status
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
// Not cached! Perform database query.
if err := dbQuery(&status); err != nil {
return nil, err
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
return &status, nil
}, keyParts...)
if err != nil {
return nil, err
}
if gtscontext.Barebones(ctx) {
// no need to fully populate.
return status, nil
}
// Further populate the status fields where applicable.
if err := s.PopulateStatus(ctx, status); err != nil {
return nil, err
}
return status, nil
}
func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) error {
var (
err error
errs = gtserror.NewMultiError(9)
)
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if status.Account == nil {
// Status author is not set, fetch from database.
status.Account, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
status.AccountID,
)
if err != nil {
errs.Appendf("error populating status author: %w", err)
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
}
}
if status.InReplyToID != "" {
if status.InReplyTo == nil {
// Status parent is not set, fetch from database.
status.InReplyTo, err = s.GetStatusByID(
gtscontext.SetBarebones(ctx),
status.InReplyToID,
)
if err != nil {
errs.Appendf("error populating status parent: %w", err)
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if status.InReplyToAccount == nil {
// Status parent author is not set, fetch from database.
status.InReplyToAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
status.InReplyToAccountID,
)
if err != nil {
errs.Appendf("error populating status parent author: %w", err)
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
}
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if status.BoostOfID != "" {
if status.BoostOf == nil {
// Status boost is not set, fetch from database.
status.BoostOf, err = s.GetStatusByID(
gtscontext.SetBarebones(ctx),
status.BoostOfID,
)
if err != nil {
errs.Appendf("error populating status boost: %w", err)
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if status.BoostOfAccount == nil {
// Status boost author is not set, fetch from database.
status.BoostOfAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
status.BoostOfAccountID,
)
if err != nil {
errs.Appendf("error populating status boost author: %w", err)
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
}
}
}
if status.PollID != "" && status.Poll == nil {
// Status poll is not set, fetch from database.
status.Poll, err = s.state.DB.GetPollByID(
gtscontext.SetBarebones(ctx),
status.PollID,
)
if err != nil {
errs.Appendf("error populating status poll: %w", err)
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if !status.AttachmentsPopulated() {
// Status attachments are out-of-date with IDs, repopulate.
status.Attachments, err = s.state.DB.GetAttachmentsByIDs(
ctx, // these are already barebones
status.AttachmentIDs,
)
if err != nil {
errs.Appendf("error populating status attachments: %w", err)
}
}
if !status.TagsPopulated() {
// Status tags are out-of-date with IDs, repopulate.
status.Tags, err = s.state.DB.GetTags(
ctx,
status.TagIDs,
)
if err != nil {
errs.Appendf("error populating status tags: %w", err)
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if !status.MentionsPopulated() {
// Status mentions are out-of-date with IDs, repopulate.
status.Mentions, err = s.state.DB.GetMentions(
ctx, // leave fully populated for now
status.MentionIDs,
)
if err != nil {
errs.Appendf("error populating status mentions: %w", err)
}
}
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if !status.EmojisPopulated() {
// Status emojis are out-of-date with IDs, repopulate.
status.Emojis, err = s.state.DB.GetEmojisByIDs(
ctx, // these are already barebones
status.EmojiIDs,
)
if err != nil {
errs.Appendf("error populating status emojis: %w", err)
}
}
if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil {
// Populate the status' expected CreatedWithApplication (not always set).
status.CreatedWithApplication, err = s.state.DB.GetApplicationByID(
ctx, // these are already barebones
status.CreatedWithApplicationID,
)
if err != nil {
errs.Appendf("error populating status application: %w", err)
}
}
return errs.Combine()
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error {
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
return s.state.Caches.GTS.Status().Store(status, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
return s.db.RunInTx(ctx, func(tx Tx) error {
// create links between this status and any emojis it uses
for _, i := range status.EmojiIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToEmoji{
StatusID: status.ID,
EmojiID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("emoji_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// create links between this status and any tags it uses
for _, i := range status.TagIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToTag{
StatusID: status.ID,
TagID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("tag_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// change the status ID of the media attachments to the new status
for _, a := range status.Attachments {
a.StatusID = status.ID
a.UpdatedAt = time.Now()
if _, err := tx.
NewUpdate().
Model(a).
Column("status_id", "updated_at").
Where("? = ?", bun.Ident("media_attachment.id"), a.ID).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// If the status is threaded, create
// link between thread and status.
if status.ThreadID != "" {
if _, err := tx.
NewInsert().
Model(&gtsmodel.ThreadToStatus{
ThreadID: status.ThreadID,
StatusID: status.ID,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("thread_id"), bun.Ident("status_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// Finally, insert the status
_, err := tx.NewInsert().Model(status).Exec(ctx)
return err
})
Improve GetRemoteStatus and db.GetStatus() logic (#174) * only fetch status parents / children if explicity requested when dereferencing Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Remove recursive DB GetStatus logic, don't fetch parent unless requested Signed-off-by: kim (grufwub) <grufwub@gmail.com> * StatusCache copies status so there are no thread-safety issues with modified status objects Signed-off-by: kim (grufwub) <grufwub@gmail.com> * remove sqlite test files Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix bugs introduced by previous commit Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix not continue on error in loop Signed-off-by: kim (grufwub) <grufwub@gmail.com> * use our own RunInTx implementation (possible fix for nested tx error) Signed-off-by: kim (grufwub) <grufwub@gmail.com> * fix cast statement to work with SQLite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * be less strict about valid status in cache Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add cache=shared ALWAYS for SQLite db instances Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix EnrichRemoteAccount when updating account fails Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add nolint tag Signed-off-by: kim (grufwub) <grufwub@gmail.com> * ensure file: prefixes the filename in sqlite addr Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add an account cache, add status author account from db Signed-off-by: kim (grufwub) <grufwub@gmail.com> * Fix incompatible SQLite query Signed-off-by: kim (grufwub) <grufwub@gmail.com> * *actually* use the new getAccount() function in accountsDB Signed-off-by: kim (grufwub) <grufwub@gmail.com> * update cache tests to use test suite Signed-off-by: kim (grufwub) <grufwub@gmail.com> * add RelationshipTestSuite, add tests for methods with changed SQL Signed-off-by: kim (grufwub) <grufwub@gmail.com>
2021-09-01 09:08:21 +00:00
})
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) error {
status.UpdatedAt = time.Now()
if len(columns) > 0 {
// If we're updating by column, ensure "updated_at" is included.
columns = append(columns, "updated_at")
}
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
return s.state.Caches.GTS.Status().Store(status, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
return s.db.RunInTx(ctx, func(tx Tx) error {
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
// create links between this status and any emojis it uses
for _, i := range status.EmojiIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToEmoji{
StatusID: status.ID,
EmojiID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("emoji_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
// create links between this status and any tags it uses
for _, i := range status.TagIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToTag{
StatusID: status.ID,
TagID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("tag_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
// change the status ID of the media attachments to the new status
for _, a := range status.Attachments {
a.StatusID = status.ID
a.UpdatedAt = time.Now()
if _, err := tx.
NewUpdate().
Model(a).
Where("? = ?", bun.Ident("media_attachment.id"), a.ID).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// If the status is threaded, create
// link between thread and status.
if status.ThreadID != "" {
if _, err := tx.
NewInsert().
Model(&gtsmodel.ThreadToStatus{
ThreadID: status.ThreadID,
StatusID: status.ID,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("thread_id"), bun.Ident("status_id")).
Exec(ctx); err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
// Finally, update the status
_, err := tx.
NewUpdate().
Model(status).
Column(columns...).
Where("? = ?", bun.Ident("status.id"), status.ID).
Exec(ctx)
return err
})
})
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) DeleteStatusByID(ctx context.Context, id string) error {
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
// Load status into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := s.GetStatusByID(
gtscontext.SetBarebones(ctx),
id,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// NOTE: even if db.ErrNoEntries is returned, we
// still run the below transaction to ensure related
// objects are appropriately deleted.
return err
}
// On return ensure status invalidated from cache.
defer s.state.Caches.GTS.Status().Invalidate("ID", id)
return s.db.RunInTx(ctx, func(tx Tx) error {
// delete links between this status and any emojis it uses
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("status_to_emojis"), bun.Ident("status_to_emoji")).
Where("? = ?", bun.Ident("status_to_emoji.status_id"), id).
Exec(ctx); err != nil {
return err
}
// delete links between this status and any tags it uses
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("status_to_tags"), bun.Ident("status_to_tag")).
Where("? = ?", bun.Ident("status_to_tag.status_id"), id).
Exec(ctx); err != nil {
return err
}
// Delete links between this status
// and any threads it was a part of.
_, err = tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("thread_to_statuses"), bun.Ident("thread_to_status")).
Where("? = ?", bun.Ident("thread_to_status.status_id"), id).
Exec(ctx)
if err != nil {
return err
}
// delete the status itself
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Where("? = ?", bun.Ident("status.id"), id).
Exec(ctx); err != nil {
return err
}
return nil
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
})
}
func (s *statusDB) GetStatusesUsingEmoji(ctx context.Context, emojiID string) ([]*gtsmodel.Status, error) {
var statusIDs []string
// SELECT all statuses using this emoji,
// using a relational table for improved perf.
if _, err := s.db.NewSelect().
Table("status_to_emojis").
Column("status_id").
Where("? = ?", bun.Ident("emoji_id"), emojiID).
Exec(ctx, &statusIDs); err != nil {
return nil, err
}
// Convert status IDs into status objects.
return s.GetStatusesByIDs(ctx, statusIDs)
}
func (s *statusDB) GetStatusParents(ctx context.Context, status *gtsmodel.Status) ([]*gtsmodel.Status, error) {
var parents []*gtsmodel.Status
for id := status.InReplyToID; id != ""; {
parent, err := s.GetStatusByID(ctx, id)
if err != nil {
return nil, err
}
// Append parent status to slice
parents = append(parents, parent)
// Set the next parent ID
id = parent.InReplyToID
}
return parents, nil
}
func (s *statusDB) GetStatusChildren(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
// Get all replies for the currently set status.
replies, err := s.GetStatusReplies(ctx, statusID)
if err != nil {
return nil, err
}
// Make estimated preallocation based on direct replies.
children := make([]*gtsmodel.Status, 0, len(replies)*2)
for _, status := range replies {
// Append status to children.
children = append(children, status)
// Further, recursively get all children for this reply.
grandChildren, err := s.GetStatusChildren(ctx, status.ID)
[chore] consolidate caching libraries (#704) * add miekg/dns dependency * set/validate accountDomain * move finger to dereferencer * totally break GetRemoteAccount * start reworking finger func a bit * start reworking getRemoteAccount a bit * move mention parts to namestring * rework webfingerget * use util function to extract webfinger parts * use accountDomain * rework finger again, final form * just a real nasty commit, the worst * remove refresh from account * use new ASRepToAccount signature * fix incorrect debug call * fix for new getRemoteAccount * rework GetRemoteAccount * start updating tests to remove repetition * break a lot of tests Move shared test logic into the testrig, rather than having it scattered all over the place. This allows us to just mock the transport controller once, and have all tests use it (unless they need not to for some other reason). * fix up tests to use main mock httpclient * webfinger only if necessary * cheeky linting with the lads * update mentionName regex recognize instance accounts * don't finger instance accounts * test webfinger part extraction * increase default worker count to 4 per cpu * don't repeat regex parsing * final search for discovered accountDomain * be more permissive in namestring lookup * add more extraction tests * simplify GetParseMentionFunc * skip long search if local account * fix broken test * consolidate to all use same caching libraries Signed-off-by: kim <grufwub@gmail.com> * perform more caching in the database layer Signed-off-by: kim <grufwub@gmail.com> * remove ASNote cache Signed-off-by: kim <grufwub@gmail.com> * update cache library, improve db tracing hooks Signed-off-by: kim <grufwub@gmail.com> * return ErrNoEntries if no account status IDs found, small formatting changes Signed-off-by: kim <grufwub@gmail.com> * fix tests, thanks tobi! Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2022-07-10 15:18:21 +00:00
if err != nil {
return nil, err
}
// Append all sub children after status.
children = append(children, grandChildren...)
}
return children, nil
}
func (s *statusDB) GetStatusReplies(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
if err != nil {
return nil, err
}
return s.GetStatusesByIDs(ctx, statusIDs)
}
func (s *statusDB) CountStatusReplies(ctx context.Context, statusID string) (int, error) {
statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
return len(statusIDs), err
}
func (s *statusDB) getStatusReplyIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.InReplyToIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status reply IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("statuses").
Column("id").
Where("? = ?", bun.Ident("in_reply_to_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, err
}
return statusIDs, nil
})
}
func (s *statusDB) GetStatusBoosts(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
if err != nil {
return nil, err
}
return s.GetStatusesByIDs(ctx, statusIDs)
}
func (s *statusDB) IsStatusBoostedBy(ctx context.Context, statusID string, accountID string) (bool, error) {
boost, err := s.GetStatusBoost(
gtscontext.SetBarebones(ctx),
statusID,
accountID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (boost != nil), nil
}
func (s *statusDB) CountStatusBoosts(ctx context.Context, statusID string) (int, error) {
statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
return len(statusIDs), err
}
func (s *statusDB) getStatusBoostIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.BoostOfIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status boost IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("statuses").
Column("id").
Where("? = ?", bun.Ident("boost_of_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, err
}
return statusIDs, nil
})
}
2023-07-25 08:34:05 +00:00
func (s *statusDB) IsStatusBookmarkedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) {
q := s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("status_bookmarks"), bun.Ident("status_bookmark")).
Where("? = ?", bun.Ident("status_bookmark.status_id"), status.ID).
Where("? = ?", bun.Ident("status_bookmark.account_id"), accountID)
2023-07-25 08:34:05 +00:00
return s.db.Exists(ctx, q)
}