Change use of Walk to WalkDir to improve disk performance (#22462)

As suggest by Go developers, use `filepath.WalkDir` instead of
`filepath.Walk` because [*Walk is less efficient than WalkDir,
introduced in Go 1.16, which avoids calling `os.Lstat` on every file or
directory visited](https://pkg.go.dev/path/filepath#Walk).

This proposition address that, in a similar way as
https://github.com/go-gitea/gitea/pull/22392 did.


Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
Felipe Leopoldo Sologuren Gutiérrez 2023-01-16 13:21:44 -03:00 committed by GitHub
parent da274380a7
commit 04c97aa364
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 32 additions and 18 deletions

View file

@ -32,11 +32,15 @@ func needsUpdate(dir, filename string) (bool, []byte) {
hasher := sha1.New() hasher := sha1.New()
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
_, _ = hasher.Write([]byte(info.Name())) info, err := d.Info()
if err != nil {
return err
}
_, _ = hasher.Write([]byte(d.Name()))
_, _ = hasher.Write([]byte(info.ModTime().String())) _, _ = hasher.Write([]byte(info.ModTime().String()))
_, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16))) _, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16)))
return nil return nil

View file

@ -225,14 +225,24 @@ func compressOldLogFile(fname string, compressionLevel int) error {
func (log *FileLogger) deleteOldLog() { func (log *FileLogger) deleteOldLog() {
dir := filepath.Dir(log.Filename) dir := filepath.Dir(log.Filename)
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) {
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r)
} }
}() }()
if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*log.Maxdays) { if err != nil {
return err
}
if d.IsDir() {
return nil
}
info, err := d.Info()
if err != nil {
return err
}
if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) {
if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) {
if err := util.Remove(path); err != nil { if err := util.Remove(path); err != nil {
returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) returnErr = fmt.Errorf("Failed to remove %s: %w", path, err)

View file

@ -173,12 +173,12 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
// Avoid walking tree if there are no globs // Avoid walking tree if there are no globs
if len(gt.Globs()) > 0 { if len(gt.Globs()) > 0 {
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil { if walkErr != nil {
return walkErr return walkErr
} }
if info.IsDir() { if d.IsDir() {
return nil return nil
} }

View file

@ -129,7 +129,7 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
// IterateObjects iterates across the objects in the local storage // IterateObjects iterates across the objects in the local storage
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error { func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error { return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
@ -141,7 +141,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
if path == l.dir { if path == l.dir {
return nil return nil
} }
if info.IsDir() { if d.IsDir() {
return nil return nil
} }
relPath, err := filepath.Rel(l.dir, path) relPath, err := filepath.Rel(l.dir, path)

View file

@ -280,17 +280,17 @@ func Repos(ctx *context.Context) {
repos := map[string]*repo_model.Repository{} repos := map[string]*repo_model.Repository{}
// We're going to iterate by pagesize. // We're going to iterate by pagesize.
root := user_model.UserPath(ctxUser.Name) root := user_model.UserPath(ctxUser.Name)
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil return nil
} }
return err return err
} }
if !info.IsDir() || path == root { if !d.IsDir() || path == root {
return nil return nil
} }
name := info.Name() name := d.Name()
if !strings.HasSuffix(name, ".git") { if !strings.HasSuffix(name, ".git") {
return filepath.SkipDir return filepath.SkipDir
} }
@ -304,7 +304,7 @@ func Repos(ctx *context.Context) {
count++ count++
return filepath.SkipDir return filepath.SkipDir
}); err != nil { }); err != nil {
ctx.ServerError("filepath.Walk", err) ctx.ServerError("filepath.WalkDir", err)
return return
} }

View file

@ -303,14 +303,16 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in
// We're going to iterate by pagesize. // We're going to iterate by pagesize.
root := filepath.Clean(setting.RepoRootPath) root := filepath.Clean(setting.RepoRootPath)
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
if !info.IsDir() || path == root { if !d.IsDir() || path == root {
return nil return nil
} }
name := d.Name()
if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) {
// Got a new user // Got a new user
if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil {
@ -318,16 +320,14 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in
} }
repoNamesToCheck = repoNamesToCheck[:0] repoNamesToCheck = repoNamesToCheck[:0]
if !globUser.Match(info.Name()) { if !globUser.Match(name) {
return filepath.SkipDir return filepath.SkipDir
} }
userName = info.Name() userName = name
return nil return nil
} }
name := info.Name()
if !strings.HasSuffix(name, ".git") { if !strings.HasSuffix(name, ".git") {
return filepath.SkipDir return filepath.SkipDir
} }