forgejo/modules/util/path.go
KN4CK3R c03e488e14
Add LFS Migration and Mirror (#14726)
* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 18:25:57 -04:00

176 lines
4.4 KiB
Go

// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import (
"errors"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"strings"
)
// EnsureAbsolutePath ensure that a path is absolute, making it
// relative to absoluteBase if necessary
func EnsureAbsolutePath(path string, absoluteBase string) string {
if filepath.IsAbs(path) {
return path
}
return filepath.Join(absoluteBase, path)
}
const notRegularFileMode os.FileMode = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
// GetDirectorySize returns the dumb disk consumption for a given path
func GetDirectorySize(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if info != nil && (info.Mode()&notRegularFileMode) == 0 {
size += info.Size()
}
return err
})
return size, err
}
// IsDir returns true if given path is a directory,
// or returns false when it's a file or does not exist.
func IsDir(dir string) (bool, error) {
f, err := os.Stat(dir)
if err == nil {
return f.IsDir(), nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
// IsFile returns true if given path is a file,
// or returns false when it's a directory or does not exist.
func IsFile(filePath string) (bool, error) {
f, err := os.Stat(filePath)
if err == nil {
return !f.IsDir(), nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
// IsExist checks whether a file or directory exists.
// It returns false when the file or directory does not exist.
func IsExist(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil || os.IsExist(err) {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool) ([]string, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err
}
defer dir.Close()
fis, err := dir.Readdir(0)
if err != nil {
return nil, err
}
statList := make([]string, 0)
for _, fi := range fis {
if strings.Contains(fi.Name(), ".DS_Store") {
continue
}
relPath := path.Join(recPath, fi.Name())
curPath := path.Join(dirPath, fi.Name())
if fi.IsDir() {
if includeDir {
statList = append(statList, relPath+"/")
}
s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks)
if err != nil {
return nil, err
}
statList = append(statList, s...)
} else if !isDirOnly {
statList = append(statList, relPath)
} else if followSymlinks && fi.Mode()&os.ModeSymlink != 0 {
link, err := os.Readlink(curPath)
if err != nil {
return nil, err
}
isDir, err := IsDir(link)
if err != nil {
return nil, err
}
if isDir {
if includeDir {
statList = append(statList, relPath+"/")
}
s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks)
if err != nil {
return nil, err
}
statList = append(statList, s...)
}
}
}
return statList, nil
}
// StatDir gathers information of given directory by depth-first.
// It returns slice of file list and includes subdirectories if enabled;
// it returns error and nil slice when error occurs in underlying functions,
// or given path is not a directory or does not exist.
//
// Slice does not include given path itself.
// If subdirectories is enabled, they will have suffix '/'.
func StatDir(rootPath string, includeDir ...bool) ([]string, error) {
if isDir, err := IsDir(rootPath); err != nil {
return nil, err
} else if !isDir {
return nil, errors.New("not a directory or does not exist: " + rootPath)
}
isIncludeDir := false
if len(includeDir) != 0 {
isIncludeDir = includeDir[0]
}
return statDir(rootPath, "", isIncludeDir, false, false)
}
// FileURLToPath extracts the path informations from a file://... url.
func FileURLToPath(u *url.URL) (string, error) {
if u.Scheme != "file" {
return "", errors.New("URL scheme is not 'file': " + u.String())
}
path := u.Path
if runtime.GOOS != "windows" {
return path, nil
}
// If it looks like there's a Windows drive letter at the beginning, strip off the leading slash.
re := regexp.MustCompile("/[A-Za-z]:/")
if re.MatchString(path) {
return path[1:], nil
}
return path, nil
}