package storage import ( "io" "io/fs" "os" "path" "syscall" "codeberg.org/gruf/go-bytes" "codeberg.org/gruf/go-pools" "codeberg.org/gruf/go-store/util" ) // DefaultDiskConfig is the default DiskStorage configuration var DefaultDiskConfig = &DiskConfig{ Overwrite: true, WriteBufSize: 4096, Transform: NopTransform(), Compression: NoCompression(), } // DiskConfig defines options to be used when opening a DiskStorage type DiskConfig struct { // Transform is the supplied key<-->path KeyTransform Transform KeyTransform // WriteBufSize is the buffer size to use when writing file streams (PutStream) WriteBufSize int // Overwrite allows overwriting values of stored keys in the storage Overwrite bool // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor } // getDiskConfig returns a valid DiskConfig for supplied ptr func getDiskConfig(cfg *DiskConfig) DiskConfig { // If nil, use default if cfg == nil { cfg = DefaultDiskConfig } // Assume nil transform == none if cfg.Transform == nil { cfg.Transform = NopTransform() } // Assume nil compress == none if cfg.Compression == nil { cfg.Compression = NoCompression() } // Assume 0 buf size == use default if cfg.WriteBufSize < 1 { cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize } // Return owned config copy return DiskConfig{ Transform: cfg.Transform, WriteBufSize: cfg.WriteBufSize, Overwrite: cfg.Overwrite, Compression: cfg.Compression, } } // DiskStorage is a Storage implementation that stores directly to a filesystem type DiskStorage struct { path string // path is the root path of this store dots int // dots is the "dotdot" count for the root store path bufp pools.BufferPool // bufp is the buffer pool for this DiskStorage config DiskConfig // cfg is the supplied configuration for this store } // OpenFile opens a DiskStorage instance for given folder path and configuration func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) // Clean provided path, ensure ends in '/' (should // be dir, this helps with file path trimming later) path = pb.Clean(path) + "/" // Get checked config config := getDiskConfig(cfg) // Attempt to open dir path file, err := os.OpenFile(path, defaultFileROFlags, defaultDirPerms) if err != nil { // If not a not-exist error, return if !os.IsNotExist(err) { return nil, err } // Attempt to make store path dirs err = os.MkdirAll(path, defaultDirPerms) if err != nil { return nil, err } // Reopen dir now it's been created file, err = os.OpenFile(path, defaultFileROFlags, defaultDirPerms) if err != nil { return nil, err } } defer file.Close() // Double check this is a dir (NOT a file!) stat, err := file.Stat() if err != nil { return nil, err } else if !stat.IsDir() { return nil, errPathIsFile } // Return new DiskStorage return &DiskStorage{ path: path, dots: util.CountDotdots(path), bufp: pools.NewBufferPool(config.WriteBufSize), config: config, }, nil } // Clean implements Storage.Clean() func (st *DiskStorage) Clean() error { return util.CleanDirs(st.path) } // ReadBytes implements Storage.ReadBytes() func (st *DiskStorage) ReadBytes(key string) ([]byte, error) { // Get stream reader for key rc, err := st.ReadStream(key) if err != nil { return nil, err } defer rc.Close() // Read all bytes and return return io.ReadAll(rc) } // ReadStream implements Storage.ReadStream() func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error) { // Get file path for key kpath, err := st.filepath(key) if err != nil { return nil, err } // Attempt to open file (replace ENOENT with our own) file, err := open(kpath, defaultFileROFlags) if err != nil { return nil, errSwapNotFound(err) } // Wrap the file in a compressor cFile, err := st.config.Compression.Reader(file) if err != nil { file.Close() // close this here, ignore error return nil, err } // Wrap compressor to ensure file close return util.ReadCloserWithCallback(cFile, func() { file.Close() }), nil } // WriteBytes implements Storage.WriteBytes() func (st *DiskStorage) WriteBytes(key string, value []byte) error { return st.WriteStream(key, bytes.NewReader(value)) } // WriteStream implements Storage.WriteStream() func (st *DiskStorage) WriteStream(key string, r io.Reader) error { // Get file path for key kpath, err := st.filepath(key) if err != nil { return err } // Ensure dirs leading up to file exist err = os.MkdirAll(path.Dir(kpath), defaultDirPerms) if err != nil { return err } // Prepare to swap error if need-be errSwap := errSwapNoop // Build file RW flags flags := defaultFileRWFlags if !st.config.Overwrite { flags |= syscall.O_EXCL // Catch + replace err exist errSwap = errSwapExist } // Attempt to open file file, err := open(kpath, flags) if err != nil { return errSwap(err) } defer file.Close() // Wrap the file in a compressor cFile, err := st.config.Compression.Writer(file) if err != nil { return err } defer cFile.Close() // Acquire write buffer buf := st.bufp.Get() defer st.bufp.Put(buf) buf.Grow(st.config.WriteBufSize) // Copy reader to file _, err = io.CopyBuffer(cFile, r, buf.B) return err } // Stat implements Storage.Stat() func (st *DiskStorage) Stat(key string) (bool, error) { // Get file path for key kpath, err := st.filepath(key) if err != nil { return false, err } // Check for file on disk return stat(kpath) } // Remove implements Storage.Remove() func (st *DiskStorage) Remove(key string) error { // Get file path for key kpath, err := st.filepath(key) if err != nil { return err } // Attempt to remove file return os.Remove(kpath) } // WalkKeys implements Storage.WalkKeys() func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error { // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) // Walk dir for entries return util.WalkDir(pb, st.path, func(kpath string, fsentry fs.DirEntry) { // Only deal with regular files if fsentry.Type().IsRegular() { // Get full item path (without root) kpath = pb.Join(kpath, fsentry.Name())[len(st.path):] // Perform provided walk function opts.WalkFn(entry(st.config.Transform.PathToKey(kpath))) } }) } // filepath checks and returns a formatted filepath for given key func (st *DiskStorage) filepath(key string) (string, error) { // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) // Calculate transformed key path key = st.config.Transform.KeyToPath(key) // Generated joined root path pb.AppendString(st.path) pb.AppendString(key) // If path is dir traversal, and traverses FURTHER // than store root, this is an error if util.CountDotdots(pb.StringPtr()) > st.dots { return "", ErrInvalidKey } return pb.String(), nil }