From 52fb9367734100847249d074e2bc17f2aa91053e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 21 Jul 2023 20:14:20 +0800 Subject: [PATCH] Serve pre-defined files in "public", add "security.txt", add CORS header for ".well-known" (#25974) Replace #25892 Close #21942 Close #25464 Major changes: 1. Serve "robots.txt" and ".well-known/security.txt" in the "public" custom path * All files in "public/.well-known" can be served, just like "public/assets" 3. Add a test for ".well-known/security.txt" 4. Simplify the "FileHandlerFunc" logic, now the paths are consistent so the code can be simpler 5. Add CORS header for ".well-known" endpoints 6. Add logs to tell users they should move some of their legacy custom public files ``` 2023/07/19 13:00:37 cmd/web.go:178:serveInstalled() [E] Found legacy public asset "img" in CustomPath. Please move it to /work/gitea/custom/public/assets/img 2023/07/19 13:00:37 cmd/web.go:182:serveInstalled() [E] Found legacy public asset "robots.txt" in CustomPath. Please move it to /work/gitea/custom/public/robots.txt ``` This PR is not breaking. --------- Co-authored-by: silverwind Co-authored-by: Giteabot --- cmd/web.go | 16 +++++++++ .../administration/customizing-gitea.en-us.md | 6 +++- modules/public/public.go | 34 ++++++------------- modules/setting/server.go | 5 --- public/.well-known/security.txt | 6 ++++ routers/install/routes.go | 2 +- routers/web/misc/misc.go | 7 ++-- routers/web/web.go | 13 +++---- tests/integration/links_test.go | 1 + 9 files changed, 50 insertions(+), 40 deletions(-) create mode 100644 public/.well-known/security.txt diff --git a/cmd/web.go b/cmd/web.go index d9aafb1fa2..dfe2091d06 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -15,9 +15,11 @@ import ( _ "net/http/pprof" // Used for debugging if enabled and a web server is running + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers/install" @@ -175,6 +177,20 @@ func serveInstalled(ctx *cli.Context) error { } } + // in old versions, user's custom web files are placed in "custom/public", and they were served as "http://domain.com/assets/xxx" + // now, Gitea only serves pre-defined files in the "custom/public" folder basing on the web root, the user should move their custom files to "custom/public/assets" + publicFiles, _ := public.AssetFS().ListFiles(".") + publicFilesSet := container.SetOf(publicFiles...) + publicFilesSet.Remove(".well-known") + publicFilesSet.Remove("assets") + publicFilesSet.Remove("robots.txt") + for _, fn := range publicFilesSet.Values() { + log.Error("Found legacy public asset %q in CustomPath. Please move it to %s/public/assets/%s", fn, setting.CustomPath, fn) + } + if _, err := os.Stat(filepath.Join(setting.CustomPath, "robots.txt")); err == nil { + log.Error(`Found legacy public asset "robots.txt" in CustomPath. Please move it to %s/public/robots.txt`, setting.CustomPath) + } + routers.InitWebInstalled(graceful.GetManager().HammerContext()) // We check that AppDataPath exists here (it should have been created during installation) diff --git a/docs/content/doc/administration/customizing-gitea.en-us.md b/docs/content/doc/administration/customizing-gitea.en-us.md index 60fcb2314b..ccc5c1bc89 100644 --- a/docs/content/doc/administration/customizing-gitea.en-us.md +++ b/docs/content/doc/administration/customizing-gitea.en-us.md @@ -56,7 +56,11 @@ is set under the "Configuration" tab on the site administration page. To make Gitea serve custom public files (like pages and images), use the folder `$GITEA_CUSTOM/public/` as the webroot. Symbolic links will be followed. -At the moment, only files in the `public/assets/` folder are served. +At the moment, only the following files are served: + +- `public/robots.txt` +- files in the `public/.well-known/` folder +- files in the `public/assets/` folder For example, a file `image.png` stored in `$GITEA_CUSTOM/public/assets/`, can be accessed with the url `http://gitea.domain.tld/assets/image.png`. diff --git a/modules/public/public.go b/modules/public/public.go index d5f0efb17a..5fbfe30a81 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -28,27 +28,15 @@ func AssetFS() *assetfs.LayeredFS { return assetfs.Layered(CustomAssets(), BuiltinAssets()) } -// AssetsHandlerFunc implements the static handler for serving custom or original assets. -func AssetsHandlerFunc(prefix string) http.HandlerFunc { +// FileHandlerFunc implements the static handler for serving files in "public" assets +func FileHandlerFunc() http.HandlerFunc { assetFS := AssetFS() - prefix = strings.TrimSuffix(prefix, "/") + "/" return func(resp http.ResponseWriter, req *http.Request) { - subPath := req.URL.Path - if !strings.HasPrefix(subPath, prefix) { - return - } - subPath = strings.TrimPrefix(subPath, prefix) - if req.Method != "GET" && req.Method != "HEAD" { resp.WriteHeader(http.StatusNotFound) return } - - if handleRequest(resp, req, assetFS, subPath) { - return - } - - resp.WriteHeader(http.StatusNotFound) + handleRequest(resp, req, assetFS, req.URL.Path) } } @@ -71,16 +59,17 @@ func setWellKnownContentType(w http.ResponseWriter, file string) { } } -func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { +func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) { // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here - f, err := fs.Open(util.PathJoinRelX("assets", file)) + f, err := fs.Open(util.PathJoinRelX(file)) if err != nil { if os.IsNotExist(err) { - return false + w.WriteHeader(http.StatusNotFound) + return } w.WriteHeader(http.StatusInternalServerError) log.Error("[Static] Open %q failed: %v", file, err) - return true + return } defer f.Close() @@ -88,17 +77,16 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, if err != nil { w.WriteHeader(http.StatusInternalServerError) log.Error("[Static] %q exists, but fails to open: %v", file, err) - return true + return } - // Try to serve index file + // need to serve index file? (no at the moment) if fi.IsDir() { w.WriteHeader(http.StatusNotFound) - return true + return } serveContent(w, req, fi, fi.ModTime(), f) - return true } type GzipBytesProvider interface { diff --git a/modules/setting/server.go b/modules/setting/server.go index 7c033bcc6b..08eb82fb3d 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -349,9 +349,4 @@ func loadServerFrom(rootCfg ConfigProvider) { default: LandingPageURL = LandingPage(landingPage) } - - HasRobotsTxt, err = util.IsFile(path.Join(CustomPath, "robots.txt")) - if err != nil { - log.Error("Unable to check if %s is a file. Error: %v", path.Join(CustomPath, "robots.txt"), err) - } } diff --git a/public/.well-known/security.txt b/public/.well-known/security.txt new file mode 100644 index 0000000000..2cae3cbea4 --- /dev/null +++ b/public/.well-known/security.txt @@ -0,0 +1,6 @@ +# This site is running a Gitea instance. +# Gitea related security problems could be reported to Gitea community. +# Site related security problems should be reported to this site's admin. +Contact: https://github.com/go-gitea/gitea/blob/main/SECURITY.md +Policy: https://github.com/go-gitea/gitea/blob/main/SECURITY.md +Preferred-Languages: en diff --git a/routers/install/routes.go b/routers/install/routes.go index ce6d41b32d..06c9d389a6 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -20,7 +20,7 @@ import ( func Routes() *web.Route { base := web.NewRoute() base.Use(common.ProtocolMiddlewares()...) - base.Methods("GET, HEAD", "/assets/*", public.AssetsHandlerFunc("/assets/")) + base.Methods("GET, HEAD", "/assets/*", public.FileHandlerFunc()) r := web.NewRoute() r.Use(common.Sessioner(), Contexter()) diff --git a/routers/web/misc/misc.go b/routers/web/misc/misc.go index 6ed3b5c3ad..54c93763f6 100644 --- a/routers/web/misc/misc.go +++ b/routers/web/misc/misc.go @@ -34,9 +34,12 @@ func DummyOK(w http.ResponseWriter, req *http.Request) { } func RobotsTxt(w http.ResponseWriter, req *http.Request) { - filePath := util.FilePathJoinAbs(setting.CustomPath, "robots.txt") + robotsTxt := util.FilePathJoinAbs(setting.CustomPath, "public/robots.txt") + if ok, _ := util.IsExist(robotsTxt); !ok { + robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt" + } httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) - http.ServeFile(w, req, filePath) + http.ServeFile(w, req, robotsTxt) } func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) { diff --git a/routers/web/web.go b/routers/web/web.go index f091bfefb8..455791ad80 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -108,7 +108,7 @@ func Routes() *web.Route { routes := web.NewRoute() routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler - routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.AssetsHandlerFunc("/assets/")) + routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.FileHandlerFunc()) routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars)) routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars)) routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png")) @@ -132,15 +132,12 @@ func Routes() *web.Route { routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...) } - if setting.HasRobotsTxt { - routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...) - } - if setting.Metrics.Enabled { prometheus.MustRegister(metrics.NewCollector()) routes.Get("/metrics", append(mid, Metrics)...) } + routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...) routes.Get("/ssh_info", misc.SSHInfo) routes.Get("/api/healthz", healthcheck.Check) @@ -336,8 +333,7 @@ func registerRoutes(m *web.Route) { // FIXME: not all routes need go through same middleware. // Especially some AJAX requests, we can reduce middleware number to improve performance. - // Routers. - // for health check + m.Get("/", Home) m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap) m.Group("/.well-known", func() { @@ -349,7 +345,8 @@ func registerRoutes(m *web.Route) { m.Get("/change-password", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/user/settings/account") }) - }) + m.Any("/*", CorsHandler(), public.FileHandlerFunc()) + }, CorsHandler()) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { diff --git a/tests/integration/links_test.go b/tests/integration/links_test.go index 9136f8f915..91655833af 100644 --- a/tests/integration/links_test.go +++ b/tests/integration/links_test.go @@ -38,6 +38,7 @@ func TestLinksNoLogin(t *testing.T) { "/user2/repo1/projects/1", "/assets/img/404.png", "/assets/img/500.png", + "/.well-known/security.txt", } for _, link := range links {