WIP: Use go:embed with server-side gzip support

Makefile

@@ -119,7 +119,7 @@ WEBPACK_CONFIGS := webpack.config.js
 WEBPACK_DEST := public/assets/js/index.js public/assets/css/index.css
 WEBPACK_DEST_ENTRIES := public/assets/js public/assets/css public/assets/fonts public/assets/img/webpack
 
-BINDATA_DEST := modules/public/bindata.go modules/options/bindata.go modules/templates/bindata.go
+BINDATA_DEST :=
 BINDATA_HASH := $(addsuffix .hash,$(BINDATA_DEST))
 
 GENERATED_GO_DEST := modules/charset/invisible_gen.go modules/charset/ambiguous_gen.go

build.go

@@ -9,9 +9,6 @@ package main
 // These libraries will not be included in a normal compilation.
 
 import (
-	// for embed
-	_ "github.com/shurcooL/vfsgen"
-
 	// for cover merge
 	_ "golang.org/x/tools/cover"
 

go.mod

@@ -93,7 +93,6 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sassoftware/go-rpmutils v0.2.0
 	github.com/sergi/go-diff v1.3.1
-	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tstranex/u2f v1.0.0
@@ -260,7 +259,6 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
@@ -301,8 +299,6 @@ require (
 
 replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1
 
-replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0
-
 replace github.com/nektos/act => gitea.com/gitea/act v0.243.4
 
 exclude github.com/gofrs/uuid v3.2.0+incompatible

go.sum

@@ -731,8 +731,6 @@ github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
 github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ=
 github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0=
-github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0 h1:F/3FfGmKdiKFa8kL3YrpZ7pe9H4l4AzA1pbaOUnRvPI=
-github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0/go.mod h1:JEfTc3+2DF9Z4PXhLLvXL42zexJyh8rIq3OzUj/0rAk=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
@@ -926,8 +924,6 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=
-github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M=
 github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
 github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d/go.mod h1:vq0tzqLRu6TS7Id0wMo2N5QzJoKedVeovOpHjnykSzY=
 github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg=

main_bindata.go

@@ -0,0 +1,26 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build bindata
+
+package main
+
+import (
+	"embed"
+	"io/fs"
+
+	"code.gitea.io/gitea/modules/migration"
+	"code.gitea.io/gitea/modules/options"
+	"code.gitea.io/gitea/modules/public"
+	"code.gitea.io/gitea/modules/templates"
+)
+
+//go:embed options public templates modules/migration/schemas
+var bindata embed.FS
+
+func init() {
+	migration.Assets, _ = fs.Sub(bindata, "modules/migration/schemas")
+	options.Assets, _ = fs.Sub(bindata, "options")
+	public.Assets, _ = fs.Sub(bindata, "public")
+	templates.Assets, _ = fs.Sub(bindata, "templates")
+}

modules/assetfs/layered.go

@@ -51,8 +51,8 @@ func Local(name, base string, sub ...string) *Layer {
 }
 
 // Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
-func Bindata(name string, fs http.FileSystem) *Layer {
-	return &Layer{name: name, fs: fs}
+func Bindata(name string, fs fs.FS) *Layer {
+	return &Layer{name: name, fs: http.FS(fs)}
 }
 
 // LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.

modules/migration/schemas_static.go

@@ -7,9 +7,12 @@ package migration
 
 import (
 	"io"
+	"io/fs"
 	"path"
 )
 
+var Assets fs.FS
+
 func openSchema(filename string) (io.ReadCloser, error) {
 	return Assets.Open(path.Base(filename))
 }

modules/options/static.go

@@ -6,9 +6,13 @@
 package options
 
 import (
+	"io/fs"
+
 	"code.gitea.io/gitea/modules/assetfs"
 )
 
+var Assets fs.FS
+
 func BuiltinAssets() *assetfs.Layer {
 	return assetfs.Bindata("builtin(bindata)", Assets)
 }

modules/public/public.go

@@ -5,11 +5,13 @@ package public
 
 import (
 	"bytes"
+	"compress/gzip"
 	"io"
 	"net/http"
 	"os"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"code.gitea.io/gitea/modules/assetfs"
@@ -28,15 +30,20 @@ func AssetFS() *assetfs.LayeredFS {
 	return assetfs.Layered(CustomAssets(), BuiltinAssets())
 }
 
+type fileHandler struct {
+	gzipContent sync.Map
+}
+
 // FileHandlerFunc implements the static handler for serving files in "public" assets
 func FileHandlerFunc() http.HandlerFunc {
 	assetFS := AssetFS()
+	fh := fileHandler{}
 	return func(resp http.ResponseWriter, req *http.Request) {
 		if req.Method != "GET" && req.Method != "HEAD" {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
-		handleRequest(resp, req, assetFS, req.URL.Path)
+		fh.handleRequest(resp, req, assetFS, req.URL.Path)
 	}
 }
 
@@ -59,7 +66,7 @@ func setWellKnownContentType(w http.ResponseWriter, file string) {
 	}
 }
 
-func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) {
+func (fh *fileHandler) handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) {
 	// actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here
 	f, err := fs.Open(util.PathJoinRelX(file))
 	if err != nil {
@@ -86,31 +93,38 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem,
 		return
 	}
 
-	serveContent(w, req, fi, fi.ModTime(), f)
-}
-
-type GzipBytesProvider interface {
-	GzipBytes() []byte
+	fh.serveContent(w, req, fi, fi.ModTime(), f)
 }
 
 // serveContent serve http content
-func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
+func (fh *fileHandler) serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
 	setWellKnownContentType(w, fi.Name())
 
 	encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
-	if encodings.Contains("gzip") {
-		// try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo)
-		if compressed, ok := fi.(GzipBytesProvider); ok {
-			rdGzip := bytes.NewReader(compressed.GzipBytes())
-			// all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name
-			// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data
-			if w.Header().Get("Content-Type") == "" {
-				w.Header().Set("Content-Type", "application/octet-stream")
-			}
-			w.Header().Set("Content-Encoding", "gzip")
-			httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip)
-			return
+	fileName := fi.Name()
+	compressible := strings.HasSuffix(fileName, ".txt") || strings.HasSuffix(fileName, ".js") || strings.HasSuffix(fileName, ".css") || strings.HasSuffix(fileName, ".svg")
+	compressible = compressible && fi.Size() > 512
+	if encodings.Contains("gzip") && compressible {
+		var compressedBytes []byte
+		if compressed, ok := fh.gzipContent.Load(fileName); !ok {
+			buf := &bytes.Buffer{}
+			c := gzip.NewWriter(buf)
+			_, _ = io.Copy(c, content)
+			_ = c.Close()
+			compressedBytes = buf.Bytes()
+			fh.gzipContent.Store(fileName, compressedBytes)
+		} else {
+			compressedBytes = compressed.([]byte)
+		}
+		rdGzip := bytes.NewReader(compressedBytes)
+		// all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name
+		// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data
+		if w.Header().Get("Content-Type") == "" {
+			w.Header().Set("Content-Type", "application/octet-stream")
 		}
+		w.Header().Set("Content-Encoding", "gzip")
+		httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip)
+		return
 	}
 
 	httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, content)

modules/public/serve_static.go

@@ -6,13 +6,14 @@
 package public
 
 import (
+	"io/fs"
 	"time"
 
 	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/timeutil"
 )
 
-var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil)
+var Assets fs.FS
 
 // GlobalModTime provide a global mod time for embedded asset files
 func GlobalModTime(filename string) time.Time {

modules/templates/static.go

@@ -6,12 +6,15 @@
 package templates
 
 import (
+	"io/fs"
 	"time"
 
 	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/timeutil"
 )
 
+var Assets fs.FS
+
 // GlobalModTime provide a global mod time for embedded asset files
 func GlobalModTime(filename string) time.Time {
 	return timeutil.GetExecutableModTime()