From 50a17578009f79d3317df27e9c4ebb9b54f3a97b Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 16 Aug 2023 12:27:04 +0800 Subject: [PATCH] fix --- Makefile | 2 +- build.go | 3 - build/generate-bindata.go | 92 -------------------------- go.mod | 4 -- go.sum | 4 -- main_bindata.go | 26 ++++++++ modules/assetfs/layered.go | 4 +- modules/migration/schemas_bindata.go | 8 --- modules/migration/schemas_static.go | 3 + modules/options/options_bindata.go | 8 --- modules/options/static.go | 4 ++ modules/public/public.go | 54 +++++++++------ modules/public/public_bindata.go | 8 --- modules/public/serve_static.go | 3 +- modules/templates/static.go | 3 + modules/templates/templates_bindata.go | 8 --- 16 files changed, 75 insertions(+), 159 deletions(-) delete mode 100644 build/generate-bindata.go create mode 100644 main_bindata.go delete mode 100644 modules/migration/schemas_bindata.go delete mode 100644 modules/options/options_bindata.go delete mode 100644 modules/public/public_bindata.go delete mode 100644 modules/templates/templates_bindata.go diff --git a/Makefile b/Makefile index 167f56c6b9266..06026a0034a1b 100644 --- a/Makefile +++ b/Makefile @@ -119,7 +119,7 @@ WEBPACK_CONFIGS := webpack.config.js WEBPACK_DEST := public/assets/js/index.js public/assets/css/index.css WEBPACK_DEST_ENTRIES := public/assets/js public/assets/css public/assets/fonts public/assets/img/webpack -BINDATA_DEST := modules/public/bindata.go modules/options/bindata.go modules/templates/bindata.go +BINDATA_DEST := BINDATA_HASH := $(addsuffix .hash,$(BINDATA_DEST)) GENERATED_GO_DEST := modules/charset/invisible_gen.go modules/charset/ambiguous_gen.go diff --git a/build.go b/build.go index 234579b514998..2dbf8acf38dbc 100644 --- a/build.go +++ b/build.go @@ -9,9 +9,6 @@ package main // These libraries will not be included in a normal compilation. import ( - // for embed - _ "github.com/shurcooL/vfsgen" - // for cover merge _ "golang.org/x/tools/cover" diff --git a/build/generate-bindata.go b/build/generate-bindata.go deleted file mode 100644 index 2fcb7c2f2a088..0000000000000 --- a/build/generate-bindata.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build ignore - -package main - -import ( - "bytes" - "crypto/sha1" - "fmt" - "log" - "net/http" - "os" - "path/filepath" - "strconv" - - "github.com/shurcooL/vfsgen" -) - -func needsUpdate(dir, filename string) (bool, []byte) { - needRegen := false - _, err := os.Stat(filename) - if err != nil { - needRegen = true - } - - oldHash, err := os.ReadFile(filename + ".hash") - if err != nil { - oldHash = []byte{} - } - - hasher := sha1.New() - - err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - info, err := d.Info() - if err != nil { - return err - } - _, _ = hasher.Write([]byte(d.Name())) - _, _ = hasher.Write([]byte(info.ModTime().String())) - _, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16))) - return nil - }) - if err != nil { - return true, oldHash - } - - newHash := hasher.Sum([]byte{}) - - if bytes.Compare(oldHash, newHash) != 0 { - return true, newHash - } - - return needRegen, newHash -} - -func main() { - if len(os.Args) < 4 { - log.Fatal("Insufficient number of arguments. Need: directory packageName filename") - } - - dir, packageName, filename := os.Args[1], os.Args[2], os.Args[3] - var useGlobalModTime bool - if len(os.Args) == 5 { - useGlobalModTime, _ = strconv.ParseBool(os.Args[4]) - } - - update, newHash := needsUpdate(dir, filename) - - if !update { - fmt.Printf("bindata for %s already up-to-date\n", packageName) - return - } - - fmt.Printf("generating bindata for %s\n", packageName) - var fsTemplates http.FileSystem = http.Dir(dir) - err := vfsgen.Generate(fsTemplates, vfsgen.Options{ - PackageName: packageName, - BuildTags: "bindata", - VariableName: "Assets", - Filename: filename, - UseGlobalModTime: useGlobalModTime, - }) - if err != nil { - log.Fatalf("%v\n", err) - } - _ = os.WriteFile(filename+".hash", newHash, 0o666) -} diff --git a/go.mod b/go.mod index b1e31c3faaff9..eec0e1cd1743e 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,6 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 github.com/sassoftware/go-rpmutils v0.2.0 github.com/sergi/go-diff v1.3.1 - github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 @@ -259,7 +258,6 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.0 // indirect github.com/spf13/afero v1.9.5 // indirect @@ -300,8 +298,6 @@ require ( replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1 -replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0 - replace github.com/nektos/act => gitea.com/gitea/act v0.243.4 exclude github.com/gofrs/uuid v3.2.0+incompatible diff --git a/go.sum b/go.sum index 5e942457a5571..b7fa7c14e9041 100644 --- a/go.sum +++ b/go.sum @@ -728,8 +728,6 @@ github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ= github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0= -github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0 h1:F/3FfGmKdiKFa8kL3YrpZ7pe9H4l4AzA1pbaOUnRvPI= -github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0/go.mod h1:JEfTc3+2DF9Z4PXhLLvXL42zexJyh8rIq3OzUj/0rAk= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= @@ -923,8 +921,6 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs= -github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d/go.mod h1:vq0tzqLRu6TS7Id0wMo2N5QzJoKedVeovOpHjnykSzY= github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg= diff --git a/main_bindata.go b/main_bindata.go new file mode 100644 index 0000000000000..eb91019da3f97 --- /dev/null +++ b/main_bindata.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build bindata + +package main + +import ( + "embed" + "io/fs" + + "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/public" + "code.gitea.io/gitea/modules/templates" +) + +//go:embed options public templates modules/migration/schemas +var bindata embed.FS + +func init() { + migration.Assets, _ = fs.Sub(bindata, "modules/migration/schemas") + options.Assets, _ = fs.Sub(bindata, "options") + public.Assets, _ = fs.Sub(bindata, "public") + templates.Assets, _ = fs.Sub(bindata, "templates") +} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index d69732f81be6b..a213ac5b6fb54 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -51,8 +51,8 @@ func Local(name, base string, sub ...string) *Layer { } // Bindata returns a new Layer with the given name, it serves files from the given bindata asset. -func Bindata(name string, fs http.FileSystem) *Layer { - return &Layer{name: name, fs: fs} +func Bindata(name string, fs fs.FS) *Layer { + return &Layer{name: name, fs: http.FS(fs)} } // LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go deleted file mode 100644 index c5db3b3461510..0000000000000 --- a/modules/migration/schemas_bindata.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package migration - -//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go index 8a0c340a65583..676ef6fcdcb11 100644 --- a/modules/migration/schemas_static.go +++ b/modules/migration/schemas_static.go @@ -7,9 +7,12 @@ package migration import ( "io" + "io/fs" "path" ) +var Assets fs.FS + func openSchema(filename string) (io.ReadCloser, error) { return Assets.Open(path.Base(filename)) } diff --git a/modules/options/options_bindata.go b/modules/options/options_bindata.go deleted file mode 100644 index 29151cb3cbc42..0000000000000 --- a/modules/options/options_bindata.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package options - -//go:generate go run ../../build/generate-bindata.go ../../options options bindata.go diff --git a/modules/options/static.go b/modules/options/static.go index 72b28e990e777..46f2aefe4539c 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -6,9 +6,13 @@ package options import ( + "io/fs" + "code.gitea.io/gitea/modules/assetfs" ) +var Assets fs.FS + func BuiltinAssets() *assetfs.Layer { return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/public/public.go b/modules/public/public.go index 5fbfe30a81c7e..8f26cc602bfee 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -5,11 +5,13 @@ package public import ( "bytes" + "compress/gzip" "io" "net/http" "os" "path" "strings" + "sync" "time" "code.gitea.io/gitea/modules/assetfs" @@ -28,15 +30,20 @@ func AssetFS() *assetfs.LayeredFS { return assetfs.Layered(CustomAssets(), BuiltinAssets()) } +type fileHandler struct { + gzipContent sync.Map +} + // FileHandlerFunc implements the static handler for serving files in "public" assets func FileHandlerFunc() http.HandlerFunc { assetFS := AssetFS() + fh := fileHandler{} return func(resp http.ResponseWriter, req *http.Request) { if req.Method != "GET" && req.Method != "HEAD" { resp.WriteHeader(http.StatusNotFound) return } - handleRequest(resp, req, assetFS, req.URL.Path) + fh.handleRequest(resp, req, assetFS, req.URL.Path) } } @@ -59,7 +66,7 @@ func setWellKnownContentType(w http.ResponseWriter, file string) { } } -func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) { +func (fh *fileHandler) handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) { // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here f, err := fs.Open(util.PathJoinRelX(file)) if err != nil { @@ -86,31 +93,38 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, return } - serveContent(w, req, fi, fi.ModTime(), f) -} - -type GzipBytesProvider interface { - GzipBytes() []byte + fh.serveContent(w, req, fi, fi.ModTime(), f) } // serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { +func (fh *fileHandler) serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { setWellKnownContentType(w, fi.Name()) encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - // try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) - if compressed, ok := fi.(GzipBytesProvider); ok { - rdGzip := bytes.NewReader(compressed.GzipBytes()) - // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name - // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data - if w.Header().Get("Content-Type") == "" { - w.Header().Set("Content-Type", "application/octet-stream") - } - w.Header().Set("Content-Encoding", "gzip") - httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip) - return + fileName := fi.Name() + compressible := strings.HasSuffix(fileName, ".txt") || strings.HasSuffix(fileName, ".js") || strings.HasSuffix(fileName, ".css") || strings.HasSuffix(fileName, ".svg") + compressible = compressible && fi.Size() > 512 + if encodings.Contains("gzip") && compressible { + var compressedBytes []byte + if compressed, ok := fh.gzipContent.Load(fileName); !ok { + buf := &bytes.Buffer{} + c := gzip.NewWriter(buf) + _, _ = io.Copy(c, content) + _ = c.Close() + compressedBytes = buf.Bytes() + fh.gzipContent.Store(fileName, compressedBytes) + } else { + compressedBytes = compressed.([]byte) + } + rdGzip := bytes.NewReader(compressedBytes) + // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name + // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data + if w.Header().Get("Content-Type") == "" { + w.Header().Set("Content-Type", "application/octet-stream") } + w.Header().Set("Content-Encoding", "gzip") + httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip) + return } httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, content) diff --git a/modules/public/public_bindata.go b/modules/public/public_bindata.go deleted file mode 100644 index 4878f88ad1d9a..0000000000000 --- a/modules/public/public_bindata.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package public - -//go:generate go run ../../build/generate-bindata.go ../../public public bindata.go true diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go index e79085021eab1..20036be583e21 100644 --- a/modules/public/serve_static.go +++ b/modules/public/serve_static.go @@ -6,13 +6,14 @@ package public import ( + "io/fs" "time" "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/timeutil" ) -var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) +var Assets fs.FS // GlobalModTime provide a global mod time for embedded asset files func GlobalModTime(filename string) time.Time { diff --git a/modules/templates/static.go b/modules/templates/static.go index b5a7e561ec065..a3f7dd07bc8a7 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -6,12 +6,15 @@ package templates import ( + "io/fs" "time" "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/timeutil" ) +var Assets fs.FS + // GlobalModTime provide a global mod time for embedded asset files func GlobalModTime(filename string) time.Time { return timeutil.GetExecutableModTime() diff --git a/modules/templates/templates_bindata.go b/modules/templates/templates_bindata.go deleted file mode 100644 index 6f1d3cf539590..0000000000000 --- a/modules/templates/templates_bindata.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package templates - -//go:generate go run ../../build/generate-bindata.go ../../templates templates bindata.go true