Serve pre-defined files in "public", add "security.txt", add CORS header for ".well-known" (#25974)

Replace #25892

Close  #21942
Close  #25464

Major changes:

1. Serve "robots.txt" and ".well-known/security.txt" in the "public"
custom path
* All files in "public/.well-known" can be served, just like
"public/assets"
3. Add a test for ".well-known/security.txt"
4. Simplify the "FileHandlerFunc" logic, now the paths are consistent so
the code can be simpler
5. Add CORS header for ".well-known" endpoints
6. Add logs to tell users they should move some of their legacy custom
public files

```
2023/07/19 13:00:37 cmd/web.go:178:serveInstalled() [E] Found legacy public asset "img" in CustomPath. Please move it to /work/gitea/custom/public/assets/img
2023/07/19 13:00:37 cmd/web.go:182:serveInstalled() [E] Found legacy public asset "robots.txt" in CustomPath. Please move it to /work/gitea/custom/public/robots.txt
```
This PR is not breaking.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
wxiaoguang 2023-07-21 20:14:20 +08:00 committed by GitHub
parent 2f0e79e639
commit 52fb936773
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 50 additions and 40 deletions

View file

@ -15,9 +15,11 @@ import (
_ "net/http/pprof" // Used for debugging if enabled and a web server is running _ "net/http/pprof" // Used for debugging if enabled and a web server is running
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers"
"code.gitea.io/gitea/routers/install" "code.gitea.io/gitea/routers/install"
@ -175,6 +177,20 @@ func serveInstalled(ctx *cli.Context) error {
} }
} }
// in old versions, user's custom web files are placed in "custom/public", and they were served as "http://domain.com/assets/xxx"
// now, Gitea only serves pre-defined files in the "custom/public" folder basing on the web root, the user should move their custom files to "custom/public/assets"
publicFiles, _ := public.AssetFS().ListFiles(".")
publicFilesSet := container.SetOf(publicFiles...)
publicFilesSet.Remove(".well-known")
publicFilesSet.Remove("assets")
publicFilesSet.Remove("robots.txt")
for _, fn := range publicFilesSet.Values() {
log.Error("Found legacy public asset %q in CustomPath. Please move it to %s/public/assets/%s", fn, setting.CustomPath, fn)
}
if _, err := os.Stat(filepath.Join(setting.CustomPath, "robots.txt")); err == nil {
log.Error(`Found legacy public asset "robots.txt" in CustomPath. Please move it to %s/public/robots.txt`, setting.CustomPath)
}
routers.InitWebInstalled(graceful.GetManager().HammerContext()) routers.InitWebInstalled(graceful.GetManager().HammerContext())
// We check that AppDataPath exists here (it should have been created during installation) // We check that AppDataPath exists here (it should have been created during installation)

View file

@ -56,7 +56,11 @@ is set under the "Configuration" tab on the site administration page.
To make Gitea serve custom public files (like pages and images), use the folder To make Gitea serve custom public files (like pages and images), use the folder
`$GITEA_CUSTOM/public/` as the webroot. Symbolic links will be followed. `$GITEA_CUSTOM/public/` as the webroot. Symbolic links will be followed.
At the moment, only files in the `public/assets/` folder are served. At the moment, only the following files are served:
- `public/robots.txt`
- files in the `public/.well-known/` folder
- files in the `public/assets/` folder
For example, a file `image.png` stored in `$GITEA_CUSTOM/public/assets/`, can be accessed with For example, a file `image.png` stored in `$GITEA_CUSTOM/public/assets/`, can be accessed with
the url `http://gitea.domain.tld/assets/image.png`. the url `http://gitea.domain.tld/assets/image.png`.

View file

@ -28,27 +28,15 @@ func AssetFS() *assetfs.LayeredFS {
return assetfs.Layered(CustomAssets(), BuiltinAssets()) return assetfs.Layered(CustomAssets(), BuiltinAssets())
} }
// AssetsHandlerFunc implements the static handler for serving custom or original assets. // FileHandlerFunc implements the static handler for serving files in "public" assets
func AssetsHandlerFunc(prefix string) http.HandlerFunc { func FileHandlerFunc() http.HandlerFunc {
assetFS := AssetFS() assetFS := AssetFS()
prefix = strings.TrimSuffix(prefix, "/") + "/"
return func(resp http.ResponseWriter, req *http.Request) { return func(resp http.ResponseWriter, req *http.Request) {
subPath := req.URL.Path
if !strings.HasPrefix(subPath, prefix) {
return
}
subPath = strings.TrimPrefix(subPath, prefix)
if req.Method != "GET" && req.Method != "HEAD" { if req.Method != "GET" && req.Method != "HEAD" {
resp.WriteHeader(http.StatusNotFound) resp.WriteHeader(http.StatusNotFound)
return return
} }
handleRequest(resp, req, assetFS, req.URL.Path)
if handleRequest(resp, req, assetFS, subPath) {
return
}
resp.WriteHeader(http.StatusNotFound)
} }
} }
@ -71,16 +59,17 @@ func setWellKnownContentType(w http.ResponseWriter, file string) {
} }
} }
func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) {
// actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here
f, err := fs.Open(util.PathJoinRelX("assets", file)) f, err := fs.Open(util.PathJoinRelX(file))
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return false w.WriteHeader(http.StatusNotFound)
return
} }
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
log.Error("[Static] Open %q failed: %v", file, err) log.Error("[Static] Open %q failed: %v", file, err)
return true return
} }
defer f.Close() defer f.Close()
@ -88,17 +77,16 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem,
if err != nil { if err != nil {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
log.Error("[Static] %q exists, but fails to open: %v", file, err) log.Error("[Static] %q exists, but fails to open: %v", file, err)
return true return
} }
// Try to serve index file // need to serve index file? (no at the moment)
if fi.IsDir() { if fi.IsDir() {
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
return true return
} }
serveContent(w, req, fi, fi.ModTime(), f) serveContent(w, req, fi, fi.ModTime(), f)
return true
} }
type GzipBytesProvider interface { type GzipBytesProvider interface {

View file

@ -349,9 +349,4 @@ func loadServerFrom(rootCfg ConfigProvider) {
default: default:
LandingPageURL = LandingPage(landingPage) LandingPageURL = LandingPage(landingPage)
} }
HasRobotsTxt, err = util.IsFile(path.Join(CustomPath, "robots.txt"))
if err != nil {
log.Error("Unable to check if %s is a file. Error: %v", path.Join(CustomPath, "robots.txt"), err)
}
} }

View file

@ -0,0 +1,6 @@
# This site is running a Gitea instance.
# Gitea related security problems could be reported to Gitea community.
# Site related security problems should be reported to this site's admin.
Contact: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
Policy: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
Preferred-Languages: en

View file

@ -20,7 +20,7 @@ import (
func Routes() *web.Route { func Routes() *web.Route {
base := web.NewRoute() base := web.NewRoute()
base.Use(common.ProtocolMiddlewares()...) base.Use(common.ProtocolMiddlewares()...)
base.Methods("GET, HEAD", "/assets/*", public.AssetsHandlerFunc("/assets/")) base.Methods("GET, HEAD", "/assets/*", public.FileHandlerFunc())
r := web.NewRoute() r := web.NewRoute()
r.Use(common.Sessioner(), Contexter()) r.Use(common.Sessioner(), Contexter())

View file

@ -34,9 +34,12 @@ func DummyOK(w http.ResponseWriter, req *http.Request) {
} }
func RobotsTxt(w http.ResponseWriter, req *http.Request) { func RobotsTxt(w http.ResponseWriter, req *http.Request) {
filePath := util.FilePathJoinAbs(setting.CustomPath, "robots.txt") robotsTxt := util.FilePathJoinAbs(setting.CustomPath, "public/robots.txt")
if ok, _ := util.IsExist(robotsTxt); !ok {
robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt"
}
httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
http.ServeFile(w, req, filePath) http.ServeFile(w, req, robotsTxt)
} }
func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) { func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) {

View file

@ -108,7 +108,7 @@ func Routes() *web.Route {
routes := web.NewRoute() routes := web.NewRoute()
routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler
routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.AssetsHandlerFunc("/assets/")) routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.FileHandlerFunc())
routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars)) routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars)) routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars))
routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png")) routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png"))
@ -132,15 +132,12 @@ func Routes() *web.Route {
routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...) routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...)
} }
if setting.HasRobotsTxt {
routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
}
if setting.Metrics.Enabled { if setting.Metrics.Enabled {
prometheus.MustRegister(metrics.NewCollector()) prometheus.MustRegister(metrics.NewCollector())
routes.Get("/metrics", append(mid, Metrics)...) routes.Get("/metrics", append(mid, Metrics)...)
} }
routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
routes.Get("/ssh_info", misc.SSHInfo) routes.Get("/ssh_info", misc.SSHInfo)
routes.Get("/api/healthz", healthcheck.Check) routes.Get("/api/healthz", healthcheck.Check)
@ -336,8 +333,7 @@ func registerRoutes(m *web.Route) {
// FIXME: not all routes need go through same middleware. // FIXME: not all routes need go through same middleware.
// Especially some AJAX requests, we can reduce middleware number to improve performance. // Especially some AJAX requests, we can reduce middleware number to improve performance.
// Routers.
// for health check
m.Get("/", Home) m.Get("/", Home)
m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap) m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap)
m.Group("/.well-known", func() { m.Group("/.well-known", func() {
@ -349,7 +345,8 @@ func registerRoutes(m *web.Route) {
m.Get("/change-password", func(ctx *context.Context) { m.Get("/change-password", func(ctx *context.Context) {
ctx.Redirect(setting.AppSubURL + "/user/settings/account") ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}) })
}) m.Any("/*", CorsHandler(), public.FileHandlerFunc())
}, CorsHandler())
m.Group("/explore", func() { m.Group("/explore", func() {
m.Get("", func(ctx *context.Context) { m.Get("", func(ctx *context.Context) {

View file

@ -38,6 +38,7 @@ func TestLinksNoLogin(t *testing.T) {
"/user2/repo1/projects/1", "/user2/repo1/projects/1",
"/assets/img/404.png", "/assets/img/404.png",
"/assets/img/500.png", "/assets/img/500.png",
"/.well-known/security.txt",
} }
for _, link := range links { for _, link := range links {