Substitute variables in path names of template repos too (#25294)

### Summary

Extend the template variable substitution to replace file paths. This
can be helpful for setting up log files & directories that should match
the repository name.

### PR Changes

 - Move files matching glob pattern when setting up repos from template
- For security, added ~escaping~ sanitization for cross-platform support
and to prevent directory traversal (thanks @silverwind for the
reference)
 - Added unit testing for escaping function 
- Fixed the integration tests for repo template generation by passing
the repo_template_id
- Updated the integration testfiles to add some variable substitution &
assert the outputs

I had to fix the existing repo template integration test and extend it
to add a check for variable substitutions.

Example:

![image](https://github.com/go-gitea/gitea/assets/12700993/621feb09-0ef3-460e-afa8-da74cd84fa4e)
This commit is contained in:
Kyle D 2023-06-20 17:14:47 -04:00 committed by GitHub
parent e50c3e8431
commit 8220e50b56
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 77 additions and 10 deletions

View file

@ -51,6 +51,8 @@ a/b/c/d.json
In any file matched by the above globs, certain variables will be expanded. In any file matched by the above globs, certain variables will be expanded.
Matching filenames and paths can also be expanded, and are conservatively sanitized to support cross-platform filesystems.
All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}` All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
| Variable | Expands To | Transformable | | Variable | Expands To | Transformable |

View file

@ -11,6 +11,7 @@ import (
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"time" "time"
@ -48,7 +49,7 @@ var defaultTransformers = []transformer{
{Name: "TITLE", Transform: util.ToTitleCase}, {Name: "TITLE", Transform: util.ToTitleCase},
} }
func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository) string { func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string {
expansions := []expansion{ expansions := []expansion{
{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers}, {Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers},
{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers}, {Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers},
@ -74,6 +75,9 @@ func generateExpansion(src string, templateRepo, generateRepo *repo_model.Reposi
return os.Expand(src, func(key string) string { return os.Expand(src, func(key string) string {
if expansion, ok := expansionMap[key]; ok { if expansion, ok := expansionMap[key]; ok {
if sanitizeFileName {
return fileNameSanitize(expansion)
}
return expansion return expansion
} }
return key return key
@ -191,10 +195,24 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
} }
if err := os.WriteFile(path, if err := os.WriteFile(path,
[]byte(generateExpansion(string(content), templateRepo, generateRepo)), []byte(generateExpansion(string(content), templateRepo, generateRepo, false)),
0o644); err != nil { 0o644); err != nil {
return err return err
} }
substPath := filepath.FromSlash(filepath.Join(tmpDirSlash,
generateExpansion(base, templateRepo, generateRepo, true)))
// Create parent subdirectories if needed or continue silently if it exists
if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil {
return err
}
// Substitute filename variables
if err := os.Rename(path, substPath); err != nil {
return err
}
break break
} }
} }
@ -353,3 +371,13 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ
return generateRepo, nil return generateRepo, nil
} }
// Sanitize user input to valid OS filenames
//
// Based on https://github.com/sindresorhus/filename-reserved-regex
// Adds ".." to prevent directory traversal
func fileNameSanitize(s string) string {
re := regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`)
return strings.TrimSpace(re.ReplaceAllString(s, "_"))
}

View file

@ -54,3 +54,14 @@ func TestGiteaTemplate(t *testing.T) {
}) })
} }
} }
func TestFileNameSanitize(t *testing.T) {
assert.Equal(t, "test_CON", fileNameSanitize("test_CON"))
assert.Equal(t, "test CON", fileNameSanitize("test CON "))
assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/.."))
assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git"))
assert.Equal(t, "_", fileNameSanitize("CON"))
assert.Equal(t, "_", fileNameSanitize("con"))
assert.Equal(t, "_", fileNameSanitize("\u0000"))
assert.Equal(t, "目标", fileNameSanitize("目标"))
}

View file

@ -0,0 +1,2 @@
x<01>ÎAJÅ0€a×9Å\@Ij2™ÂCÞwž"™Ìh±i¥<69>ÞÞ·qïö‡~Þ{_ ¦ ì<10>çæ+cÔ)M<>•³* rȉSD&ŠM³û*‡l¥pm*³Ž5fE_ªP ˜´D™QCËÉ•aûo?«À+\>ÛèûfÛ¸¾÷²¬O¼÷HH9G"xôÑ{w¯÷;“ÿ8
iþsîÖœ£ž¶Ø0<C398>ï²9Ý/å IH

View file

@ -1 +1 @@
aacbdfe9e1c4b47f60abe81849045fa4e96f1d75 2a83b349fa234131fc5db6f2a0498d3f4d3d6038

View file

@ -7,16 +7,18 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/tests" "code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder { func testRepoGenerate(t *testing.T, session *TestSession, templateID, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder {
generateOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: generateOwnerName}) generateOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: generateOwnerName})
// Step0: check the existence of the generated repo // Step0: check the existence of the generated repo
@ -41,16 +43,38 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem
_, exists = htmlDoc.doc.Find(fmt.Sprintf(".owner.dropdown .item[data-value=\"%d\"]", generateOwner.ID)).Attr("data-value") _, exists = htmlDoc.doc.Find(fmt.Sprintf(".owner.dropdown .item[data-value=\"%d\"]", generateOwner.ID)).Attr("data-value")
assert.True(t, exists, fmt.Sprintf("Generate owner '%s' is not present in select box", generateOwnerName)) assert.True(t, exists, fmt.Sprintf("Generate owner '%s' is not present in select box", generateOwnerName))
req = NewRequestWithValues(t, "POST", link, map[string]string{ req = NewRequestWithValues(t, "POST", link, map[string]string{
"_csrf": htmlDoc.GetCSRF(), "_csrf": htmlDoc.GetCSRF(),
"uid": fmt.Sprintf("%d", generateOwner.ID), "uid": fmt.Sprintf("%d", generateOwner.ID),
"repo_name": generateRepoName, "repo_name": generateRepoName,
"git_content": "true", "repo_template": templateID,
"git_content": "true",
}) })
session.MakeRequest(t, req, http.StatusSeeOther) session.MakeRequest(t, req, http.StatusSeeOther)
// Step4: check the existence of the generated repo // Step4: check the existence of the generated repo
req = NewRequestf(t, "GET", "/%s/%s", generateOwnerName, generateRepoName) req = NewRequestf(t, "GET", "/%s/%s", generateOwnerName, generateRepoName)
session.MakeRequest(t, req, http.StatusOK)
// Step5: check substituted values in Readme
req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/README.md", generateOwnerName, generateRepoName)
resp = session.MakeRequest(t, req, http.StatusOK) resp = session.MakeRequest(t, req, http.StatusOK)
body := fmt.Sprintf(`# %s Readme
Owner: %s
Link: /%s/%s
Clone URL: %s%s/%s.git`,
generateRepoName,
strings.ToUpper(generateOwnerName),
generateOwnerName,
generateRepoName,
setting.AppURL,
generateOwnerName,
generateRepoName)
assert.Equal(t, body, resp.Body.String())
// Step6: check substituted values in substituted file path ${REPO_NAME}
req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/%s.log", generateOwnerName, generateRepoName, generateRepoName)
resp = session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, generateRepoName, resp.Body.String())
return resp return resp
} }
@ -58,11 +82,11 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem
func TestRepoGenerate(t *testing.T) { func TestRepoGenerate(t *testing.T) {
defer tests.PrepareTestEnv(t)() defer tests.PrepareTestEnv(t)()
session := loginUser(t, "user1") session := loginUser(t, "user1")
testRepoGenerate(t, session, "user27", "template1", "user1", "generated1") testRepoGenerate(t, session, "44", "user27", "template1", "user1", "generated1")
} }
func TestRepoGenerateToOrg(t *testing.T) { func TestRepoGenerateToOrg(t *testing.T) {
defer tests.PrepareTestEnv(t)() defer tests.PrepareTestEnv(t)()
session := loginUser(t, "user2") session := loginUser(t, "user2")
testRepoGenerate(t, session, "user27", "template1", "user2", "generated2") testRepoGenerate(t, session, "44", "user27", "template1", "user2", "generated2")
} }