[bugfix] KVStore doesn't like lost+found directory (#972)

* bump go-store version to v2.0.5, init kv.KVStore without initial clean (as we are using for storage, not as a key-value store)

Signed-off-by: kim <grufwub@gmail.com>

* remove newline

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2022-11-06 12:30:08 +00:00 committed by GitHub
parent 298a7ad21b
commit 05a8baa53a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 25 additions and 48 deletions

View file

@ -104,7 +104,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
typeConverter := typeutils.NewConverter(dbService) typeConverter := typeutils.NewConverter(dbService)
// Open the storage backend // Open the storage backend
storage, err := gtsstorage.AutoConfig() storage, err := gtsstorage.AutoConfig()
if err != nil { if err != nil {
return fmt.Errorf("error creating storage backend: %w", err) return fmt.Errorf("error creating storage backend: %w", err)

2
go.mod
View file

@ -12,7 +12,7 @@ require (
codeberg.org/gruf/go-logger/v2 v2.2.1 codeberg.org/gruf/go-logger/v2 v2.2.1
codeberg.org/gruf/go-mutexes v1.1.3 codeberg.org/gruf/go-mutexes v1.1.3
codeberg.org/gruf/go-runners v1.3.1 codeberg.org/gruf/go-runners v1.3.1
codeberg.org/gruf/go-store/v2 v2.0.4 codeberg.org/gruf/go-store/v2 v2.0.5
github.com/buckket/go-blurhash v1.1.0 github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.4.0 github.com/coreos/go-oidc/v3 v3.4.0
github.com/disintegration/imaging v1.6.2 github.com/disintegration/imaging v1.6.2

4
go.sum
View file

@ -95,8 +95,8 @@ codeberg.org/gruf/go-runners v1.3.1 h1:d/OQMMMiA6yPaDSbSr0/Jc+lucWmm7AiAZjWffpNK
codeberg.org/gruf/go-runners v1.3.1/go.mod h1:rl0EdZNozkRMb21DAtOL5L4oTfmslYQdZgq2RMMc/H4= codeberg.org/gruf/go-runners v1.3.1/go.mod h1:rl0EdZNozkRMb21DAtOL5L4oTfmslYQdZgq2RMMc/H4=
codeberg.org/gruf/go-sched v1.1.1 h1:YtLSQhpypzuD3HTup5oF7LLWB79gTL4nqW06kH4Vwks= codeberg.org/gruf/go-sched v1.1.1 h1:YtLSQhpypzuD3HTup5oF7LLWB79gTL4nqW06kH4Vwks=
codeberg.org/gruf/go-sched v1.1.1/go.mod h1:SRcdP/5qim+EBT3n3r4aUra1C30yPqV4OJOXuqvgdQM= codeberg.org/gruf/go-sched v1.1.1/go.mod h1:SRcdP/5qim+EBT3n3r4aUra1C30yPqV4OJOXuqvgdQM=
codeberg.org/gruf/go-store/v2 v2.0.4 h1:CpGkJUz7qINh3krjtR6dKmKwJKjprbEKKq4dWF+AemI= codeberg.org/gruf/go-store/v2 v2.0.5 h1:AbOka6LkyT9jobPYfK3h5f5dPqx5AjFPhaaqdOWkGyA=
codeberg.org/gruf/go-store/v2 v2.0.4/go.mod h1:vKId86ET4ZzG1tE1dMNkfV66rZkcsyqt64UhKt6EYfc= codeberg.org/gruf/go-store/v2 v2.0.5/go.mod h1:vKId86ET4ZzG1tE1dMNkfV66rZkcsyqt64UhKt6EYfc=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

View file

@ -20,43 +20,13 @@ package storage
import ( import (
"context" "context"
"io"
"net/url" "net/url"
"codeberg.org/gruf/go-store/v2/kv" "codeberg.org/gruf/go-store/v2/kv"
"codeberg.org/gruf/go-store/v2/storage"
) )
type Local struct { type Local struct {
KVStore *kv.KVStore *kv.KVStore
}
func (l *Local) Get(ctx context.Context, key string) ([]byte, error) {
return l.KVStore.Get(ctx, key)
}
func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
return l.KVStore.GetStream(ctx, key)
}
func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error {
err := l.KVStore.PutStream(ctx, key, r)
if err == storage.ErrAlreadyExists {
return ErrAlreadyExists
}
return err
}
func (l *Local) Put(ctx context.Context, key string, value []byte) error {
err := l.KVStore.Put(ctx, key, value)
if err == storage.ErrAlreadyExists {
return ErrAlreadyExists
}
return err
}
func (l *Local) Delete(ctx context.Context, key string) error {
return l.KVStore.Delete(ctx, key)
} }
func (l *Local) URL(ctx context.Context, key string) *url.URL { func (l *Local) URL(ctx context.Context, key string) *url.URL {

View file

@ -33,8 +33,10 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/config"
) )
var ErrNotSupported = errors.New("driver does not suppport functionality") var (
var ErrAlreadyExists = errors.New("storage key already exists") ErrNotSupported = errors.New("driver does not suppport functionality")
ErrAlreadyExists = storage.ErrAlreadyExists
)
// Driver implements the functionality to store and retrieve blobs // Driver implements the functionality to store and retrieve blobs
// (images,video,audio) // (images,video,audio)
@ -59,19 +61,19 @@ func AutoConfig() (Driver, error) {
} }
return NewS3(mc, config.GetStorageS3BucketName()), nil return NewS3(mc, config.GetStorageS3BucketName()), nil
case "local": case "local":
storageBasePath := config.GetStorageLocalBasePath() basePath := config.GetStorageLocalBasePath()
storage, err := kv.OpenDisk(storageBasePath, &storage.DiskConfig{ disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{
// Put the store lockfile in the storage dir itself. // Put the store lockfile in the storage dir itself.
// Normally this would not be safe, since we could end up // Normally this would not be safe, since we could end up
// overwriting the lockfile if we store a file called 'store.lock'. // overwriting the lockfile if we store a file called 'store.lock'.
// However, in this case it's OK because the keys are set by // However, in this case it's OK because the keys are set by
// GtS and not the user, so we know we're never going to overwrite it. // GtS and not the user, so we know we're never going to overwrite it.
LockFile: path.Join(storageBasePath, "store.lock"), LockFile: path.Join(basePath, "store.lock"),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("error creating storage backend: %s", err) return nil, fmt.Errorf("error openingdisk storage: %v", err)
} }
return &Local{KVStore: storage}, nil return &Local{kv.New(disk)}, nil
} }
return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend()) return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())
} }

View file

@ -38,10 +38,7 @@ func OpenBlock(path string, cfg *storage.BlockConfig) (*KVStore, error) {
} }
func OpenMemory(overwrites bool) *KVStore { func OpenMemory(overwrites bool) *KVStore {
return &KVStore{ return New(storage.OpenMemory(100, overwrites))
mu: mutexes.NewMap(-1, -1),
st: storage.OpenMemory(100, overwrites),
}
} }
func OpenS3(endpoint string, bucket string, cfg *storage.S3Config) (*KVStore, error) { func OpenS3(endpoint string, bucket string, cfg *storage.S3Config) (*KVStore, error) {
@ -55,6 +52,7 @@ func OpenS3(endpoint string, bucket string, cfg *storage.S3Config) (*KVStore, er
return OpenStorage(storage) return OpenStorage(storage)
} }
// OpenStorage will return a new KVStore instance based on Storage, performing an initial storage.Clean().
func OpenStorage(storage storage.Storage) (*KVStore, error) { func OpenStorage(storage storage.Storage) (*KVStore, error) {
// Perform initial storage clean // Perform initial storage clean
err := storage.Clean(context.Background()) err := storage.Clean(context.Background())
@ -63,10 +61,18 @@ func OpenStorage(storage storage.Storage) (*KVStore, error) {
} }
// Return new KVStore // Return new KVStore
return New(storage), nil
}
// New will simply return a new KVStore instance based on Storage.
func New(storage storage.Storage) *KVStore {
if storage == nil {
panic("nil storage")
}
return &KVStore{ return &KVStore{
mu: mutexes.NewMap(-1, -1), mu: mutexes.NewMap(-1, -1),
st: storage, st: storage,
}, nil }
} }
// RLock acquires a read-lock on supplied key, returning unlock function. // RLock acquires a read-lock on supplied key, returning unlock function.

2
vendor/modules.txt vendored
View file

@ -50,7 +50,7 @@ codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-sched v1.1.1 # codeberg.org/gruf/go-sched v1.1.1
## explicit; go 1.19 ## explicit; go 1.19
codeberg.org/gruf/go-sched codeberg.org/gruf/go-sched
# codeberg.org/gruf/go-store/v2 v2.0.4 # codeberg.org/gruf/go-store/v2 v2.0.5
## explicit; go 1.19 ## explicit; go 1.19
codeberg.org/gruf/go-store/v2/kv codeberg.org/gruf/go-store/v2/kv
codeberg.org/gruf/go-store/v2/storage codeberg.org/gruf/go-store/v2/storage