gotosocial/vendor/codeberg.org/gruf/go-store/v2/storage/memory.go
kim 5318054808
[performance] media processing improvements (#1288)
* media processor consolidation and reformatting, reduce amount of required syscalls

Signed-off-by: kim <grufwub@gmail.com>

* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix duration not being set, fix mp4 test expecting error

Signed-off-by: kim <grufwub@gmail.com>

* fix test expecting media files with different extension

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* fix expected storage paths in tests, update expected test thumbnails

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* fix cached presigned s3 url fetching

Signed-off-by: kim <grufwub@gmail.com>

* fix tests

Signed-off-by: kim <grufwub@gmail.com>

* fix test models

Signed-off-by: kim <grufwub@gmail.com>

* update media processing to use sync.Once{} for concurrency protection

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* fix passing in KVStore GetStream() as stream to PutStream()

Signed-off-by: kim <grufwub@gmail.com>

* fix unlocks of storage keys

Signed-off-by: kim <grufwub@gmail.com>

* whoops, return the error...

Signed-off-by: kim <grufwub@gmail.com>

* pour one out for tobi's code <3

Signed-off-by: kim <grufwub@gmail.com>

* add back the byte slurping code

Signed-off-by: kim <grufwub@gmail.com>

* check for both ErrUnexpectedEOF and EOF

Signed-off-by: kim <grufwub@gmail.com>

* add back links to file format header information

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
2023-01-11 12:13:13 +01:00

229 lines
4.4 KiB
Go

package storage
import (
"context"
"io"
"sync/atomic"
"codeberg.org/gruf/go-bytes"
"codeberg.org/gruf/go-iotools"
"github.com/cornelk/hashmap"
)
// MemoryStorage is a storage implementation that simply stores key-value
// pairs in a Go map in-memory. The map is protected by a mutex.
type MemoryStorage struct {
ow bool // overwrites
fs *hashmap.Map[string, []byte]
st uint32
}
// OpenMemory opens a new MemoryStorage instance with internal map starting size.
func OpenMemory(size int, overwrites bool) *MemoryStorage {
if size <= 0 {
size = 8
}
return &MemoryStorage{
fs: hashmap.NewSized[string, []byte](uintptr(size)),
ow: overwrites,
}
}
// Clean implements Storage.Clean().
func (st *MemoryStorage) Clean(ctx context.Context) error {
// Check store open
if st.closed() {
return ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
}
return nil
}
// ReadBytes implements Storage.ReadBytes().
func (st *MemoryStorage) ReadBytes(ctx context.Context, key string) ([]byte, error) {
// Check store open
if st.closed() {
return nil, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return nil, err
}
// Check for key in store
b, ok := st.fs.Get(key)
if !ok {
return nil, ErrNotFound
}
// Create return copy
return copyb(b), nil
}
// ReadStream implements Storage.ReadStream().
func (st *MemoryStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) {
// Check store open
if st.closed() {
return nil, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return nil, err
}
// Check for key in store
b, ok := st.fs.Get(key)
if !ok {
return nil, ErrNotFound
}
// Create io.ReadCloser from 'b' copy
r := bytes.NewReader(copyb(b))
return iotools.NopReadCloser(r), nil
}
// WriteBytes implements Storage.WriteBytes().
func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) (int, error) {
// Check store open
if st.closed() {
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return 0, err
}
// Check for key that already exists
if _, ok := st.fs.Get(key); ok && !st.ow {
return 0, ErrAlreadyExists
}
// Write key copy to store
st.fs.Set(key, copyb(b))
return len(b), nil
}
// WriteStream implements Storage.WriteStream().
func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
// Check store open
if st.closed() {
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return 0, err
}
// Check for key that already exists
if _, ok := st.fs.Get(key); ok && !st.ow {
return 0, ErrAlreadyExists
}
// Read all from reader
b, err := io.ReadAll(r)
if err != nil {
return 0, err
}
// Write key to store
st.fs.Set(key, b)
return int64(len(b)), nil
}
// Stat implements Storage.Stat().
func (st *MemoryStorage) Stat(ctx context.Context, key string) (bool, error) {
// Check store open
if st.closed() {
return false, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return false, err
}
// Check for key in store
_, ok := st.fs.Get(key)
return ok, nil
}
// Remove implements Storage.Remove().
func (st *MemoryStorage) Remove(ctx context.Context, key string) error {
// Check store open
if st.closed() {
return ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
}
// Attempt to delete key
ok := st.fs.Del(key)
if !ok {
return ErrNotFound
}
return nil
}
// WalkKeys implements Storage.WalkKeys().
func (st *MemoryStorage) WalkKeys(ctx context.Context, opts WalkKeysOptions) error {
// Check store open
if st.closed() {
return ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
}
var err error
// Nil check func
_ = opts.WalkFn
// Pass each key in map to walk function
st.fs.Range(func(key string, val []byte) bool {
err = opts.WalkFn(ctx, Entry{
Key: key,
Size: int64(len(val)),
})
return (err == nil)
})
return err
}
// Close implements Storage.Close().
func (st *MemoryStorage) Close() error {
atomic.StoreUint32(&st.st, 1)
return nil
}
// closed returns whether MemoryStorage is closed.
func (st *MemoryStorage) closed() bool {
return (atomic.LoadUint32(&st.st) == 1)
}
// copyb returns a copy of byte-slice b.
func copyb(b []byte) []byte {
if b == nil {
return nil
}
p := make([]byte, len(b))
_ = copy(p, b)
return p
}