Improve vfsgen to not unzip bindata files but send to browser directly (#7109)

* Don't unzip files from bindata but send to browser directly

* remove dependent for httpgzip

* Add tests for parseAcceptEncoding

* Update docs for ENABLE_GZIP

* Fix bug

* Fix bug

Co-authored-by: zeripath <art27@cantab.net>

custom/conf/app.example.ini

@@ -361,7 +361,7 @@ KEY_FILE = https/key.pem
 STATIC_ROOT_PATH =
 ; Default path for App data
 APP_DATA_PATH = data
-; Application level GZIP support
+; Enable gzip compression for runtime-generated content, static resources excluded
 ENABLE_GZIP = false
 ; Application profiling (memory and cpu)
 ; For "web" command it listens on localhost:6060

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -264,7 +264,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path.
 - `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data.
 - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. Note that this cache is disabled when `RUN_MODE` is "dev".
-- `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
+- `ENABLE_GZIP`: **false**: Enable gzip compression for runtime-generated content, static resources excluded.
 - `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)_<username>_<temporary id>`
 - `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service
 - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -70,7 +70,7 @@ menu:
 - `KEY_FILE`: 启用HTTPS的密钥文件。
 - `STATIC_ROOT_PATH`: 存放模板和静态文件的根目录，默认是 Gitea 的根目录。
 - `STATIC_CACHE_TIME`: **6h**: 静态资源文件，包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
-- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
+- `ENABLE_GZIP`: 启用实时生成的数据启用 GZIP 压缩，不包括静态资源。
 - `LANDING_PAGE`: 未登录用户的默认页面，可选 `home` 或 `explore`。
 
 - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。

modules/public/dynamic.go

@@ -6,9 +6,19 @@
 
 package public
 
-import "net/http"
+import (
+	"io"
+	"net/http"
+	"os"
+	"time"
+)
 
 // Static implements the macaron static handler for serving assets.
 func Static(opts *Options) func(next http.Handler) http.Handler {
 	return opts.staticHandler(opts.Directory)
 }
+
+// ServeContent serve http content
+func ServeContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
+	http.ServeContent(w, req, fi.Name(), modtime, content)
+}

modules/public/public.go

@@ -87,6 +87,16 @@ func (opts *Options) staticHandler(dir string) func(next http.Handler) http.Hand
 	}
 }
 
+// parseAcceptEncoding parse Accept-Encoding: deflate, gzip;q=1.0, *;q=0.5 as compress methods
+func parseAcceptEncoding(val string) map[string]bool {
+	parts := strings.Split(val, ";")
+	var types = make(map[string]bool)
+	for _, v := range strings.Split(parts[0], ",") {
+		types[strings.TrimSpace(v)] = true
+	}
+	return types
+}
+
 func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Options) bool {
 	if req.Method != "GET" && req.Method != "HEAD" {
 		return false
@@ -157,6 +167,6 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Optio
 		return true
 	}
 
-	http.ServeContent(w, req, file, fi.ModTime(), f)
+	ServeContent(w, req, fi, fi.ModTime(), f)
 	return true
 }

modules/public/public_test.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package public
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseAcceptEncoding(t *testing.T) {
+	var kases = []struct {
+		Header   string
+		Expected map[string]bool
+	}{
+		{
+			Header: "deflate, gzip;q=1.0, *;q=0.5",
+			Expected: map[string]bool{
+				"deflate": true,
+				"gzip":    true,
+			},
+		},
+		{
+			Header: " gzip, deflate, br",
+			Expected: map[string]bool{
+				"deflate": true,
+				"gzip":    true,
+				"br":      true,
+			},
+		},
+	}
+
+	for _, kase := range kases {
+		t.Run(kase.Header, func(t *testing.T) {
+			assert.EqualValues(t, kase.Expected, parseAcceptEncoding(kase.Header))
+		})
+	}
+}

modules/public/static.go

@@ -7,8 +7,17 @@
 package public
 
 import (
+	"bytes"
+	"compress/gzip"
+	"io"
 	"io/ioutil"
+	"mime"
 	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
 )
 
 // Static implements the macaron static handler for serving assets.
@@ -49,3 +58,34 @@ func AssetIsDir(name string) (bool, error) {
 		}
 	}
 }
+
+// ServeContent serve http content
+func ServeContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
+	encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
+	if encodings["gzip"] {
+		if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok {
+			rd := bytes.NewReader(cf.GzipBytes())
+			w.Header().Set("Content-Encoding", "gzip")
+			ctype := mime.TypeByExtension(filepath.Ext(fi.Name()))
+			if ctype == "" {
+				// read a chunk to decide between utf-8 text and binary
+				var buf [512]byte
+				grd, _ := gzip.NewReader(rd)
+				n, _ := io.ReadFull(grd, buf[:])
+				ctype = http.DetectContentType(buf[:n])
+				_, err := rd.Seek(0, io.SeekStart) // rewind to output whole file
+				if err != nil {
+					log.Error("rd.Seek error: %v", err)
+					http.Error(w, http.StatusText(500), 500)
+					return
+				}
+			}
+			w.Header().Set("Content-Type", ctype)
+			http.ServeContent(w, req, fi.Name(), modtime, rd)
+			return
+		}
+	}
+
+	http.ServeContent(w, req, fi.Name(), modtime, content)
+	return
+}