Migrate from bindata to go:embed

go:embed is part of the standard library and does not require an extra
go:generate step.

Migrate all usages of bindata to go:embed.

Make embedded the new default (previously required bindata tag).

Add new tag "servedynamic" which serves from filesystem (old !bindata).

Accept-Encoding compression has been dropped. The assets in go:embed are not
available in a compressed form. The compression could be enabled again by adding
a compress middleware: https://github.com/vearutop/statigz

Drop vfsgen dependency (no longer required).

Fixes #17352

Signed-off-by: Christian Stewart <christian@paral.in>

Author: paralin

.gitignore

@@ -36,15 +36,6 @@ _testmain.go
 coverage.all
 cpu.out
 
-/modules/migration/bindata.go
-/modules/migration/bindata.go.hash
-/modules/options/bindata.go
-/modules/options/bindata.go.hash
-/modules/public/bindata.go
-/modules/public/bindata.go.hash
-/modules/templates/bindata.go
-/modules/templates/bindata.go.hash
-
 *.db
 *.log
 

Dockerfile

@@ -6,7 +6,7 @@ ENV GOPROXY ${GOPROXY:-direct}
 
 ARG GITEA_VERSION
 ARG TAGS="sqlite sqlite_unlock_notify"
-ENV TAGS "bindata timetzdata $TAGS"
+ENV TAGS "timetzdata $TAGS"
 ARG CGO_EXTRA_CFLAGS
 
 #Build deps

Dockerfile.rootless

@@ -6,7 +6,7 @@ ENV GOPROXY ${GOPROXY:-direct}
 
 ARG GITEA_VERSION
 ARG TAGS="sqlite sqlite_unlock_notify"
-ENV TAGS "bindata timetzdata $TAGS"
+ENV TAGS "timetzdata $TAGS"
 ARG CGO_EXTRA_CFLAGS
 
 #Build deps
@@ -73,4 +73,3 @@ WORKDIR /var/lib/gitea
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/docker-entrypoint.sh"]
 CMD []
-

Makefile

@@ -110,9 +110,6 @@ WEBPACK_CONFIGS := webpack.config.js
 WEBPACK_DEST := public/js/index.js public/css/index.css
 WEBPACK_DEST_ENTRIES := public/js public/css public/fonts public/img/webpack public/serviceworker.js
 
-BINDATA_DEST := modules/public/bindata.go modules/options/bindata.go modules/templates/bindata.go
-BINDATA_HASH := $(addsuffix .hash,$(BINDATA_DEST))
-
 GENERATED_GO_DEST := modules/charset/invisible_gen.go modules/charset/ambiguous_gen.go
 
 SVG_DEST_DIR := public/img/svg
@@ -134,14 +131,8 @@ GO_DIRS := cmd tests models modules routers build services tools
 WEB_DIRS := web_src/js web_src/less
 
 GO_SOURCES := $(wildcard *.go)
-GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go)
+GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go")
 GO_SOURCES += $(GENERATED_GO_DEST)
-GO_SOURCES_NO_BINDATA := $(GO_SOURCES)
-
-ifeq ($(filter $(TAGS_SPLIT),bindata),bindata)
-	GO_SOURCES += $(BINDATA_DEST)
-	GENERATED_GO_DEST += $(BINDATA_DEST)
-endif
 
 # Force installation of playwright dependencies by setting this flag
 ifdef DEPS_PLAYWRIGHT
@@ -253,7 +244,7 @@ clean-all: clean
 .PHONY: clean
 clean:
 	$(GO) clean -i ./...
-	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA_DEST) $(BINDATA_HASH) \
+	rm -rf $(EXECUTABLE) $(DIST) \
 		integrations*.test \
 		e2e*.test \
 		tests/integration/gitea-integration-pgsql/ tests/integration/gitea-integration-mysql/ tests/integration/gitea-integration-mysql8/ tests/integration/gitea-integration-sqlite/ \
@@ -301,7 +292,7 @@ endif
 .PHONY: generate-swagger
 generate-swagger: $(SWAGGER_SPEC)
 
-$(SWAGGER_SPEC): $(GO_SOURCES_NO_BINDATA)
+$(SWAGGER_SPEC): $(GO_SOURCES)
 	$(GO) run $(SWAGGER_PACKAGE) generate spec -x "$(SWAGGER_EXCLUDE)" -o './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) '$(SWAGGER_SPEC_S_TMPL)' './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) $(SWAGGER_NEWLINE_COMMAND) './$(SWAGGER_SPEC)'
@@ -950,11 +941,11 @@ editorconfig-checker:
 .PHONY: docker
 docker:
 	docker build --disable-content-trust=false -t $(DOCKER_REF) .
-# support also build args docker build --build-arg GITEA_VERSION=v1.2.3 --build-arg TAGS="bindata sqlite sqlite_unlock_notify"  .
+# support also build args docker build --build-arg GITEA_VERSION=v1.2.3 --build-arg TAGS="sqlite sqlite_unlock_notify"  .
 
 .PHONY: docker-build
 docker-build:
-	docker run -ti --rm -v "$(CURDIR):/srv/app/src/code.gitea.io/gitea" -w /srv/app/src/code.gitea.io/gitea -e TAGS="bindata $(TAGS)" LDFLAGS="$(LDFLAGS)" CGO_EXTRA_CFLAGS="$(CGO_EXTRA_CFLAGS)" webhippie/golang:edge make clean build
+	docker run -ti --rm -v "$(CURDIR):/srv/app/src/code.gitea.io/gitea" -w /srv/app/src/code.gitea.io/gitea -e TAGS="$(TAGS)" LDFLAGS="$(LDFLAGS)" CGO_EXTRA_CFLAGS="$(CGO_EXTRA_CFLAGS)" webhippie/golang:edge make clean build
 
 # This endif closes the if at the top of the file
 endif

README.md

@@ -71,11 +71,11 @@ This project has been
 
 From the root of the source tree, run:
 
-    TAGS="bindata" make build
+    make build
 
 or if SQLite support is required:
 
-    TAGS="bindata sqlite sqlite_unlock_notify" make build
+    TAGS="sqlite sqlite_unlock_notify" make build
 
 The `build` target is split into two sub-targets:
 

build.go

@@ -1,7 +1,6 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-
 //go:build vendor
 
 package main
@@ -10,9 +9,6 @@ package main
 // These libraries will not be included in a normal compilation.
 
 import (
-	// for embed
-	_ "github.com/shurcooL/vfsgen"
-
 	// for cover merge
 	_ "golang.org/x/tools/cover"
 

build/code-batch-process.go

@@ -69,7 +69,6 @@ func newFileCollector(fileFilter string, batchSize int) (*fileCollector, error)
 		}
 		co.includePatterns = append(co.includePatterns, regexp.MustCompile(`.*\.go$`))
 
-		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`.*\bbindata\.go$`))
 		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/gitea-repositories-meta`))
 		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/integration/migration-test`))
 		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`modules/git/tests`))

build/generate-bindata.go

@@ -1,7 +1,7 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-//go:build bindata
+//go:build !servedynamic
 
 package cmd
 
@@ -89,8 +89,7 @@ var (
 		},
 	}
 
-	sections map[string]*section
-	assets   []asset
+	assets []asset
 )
 
 type section struct {
@@ -227,7 +226,7 @@ func runExtractDo(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("%s: %s", destdir, err)
 	} else if !fi.IsDir() {
-		return fmt.Errorf("%s is not a directory.", destdir)
+		return fmt.Errorf("%s is not a directory", destdir)
 	}
 
 	fmt.Printf("Extracting to %s:\n", destdir)
@@ -324,11 +323,11 @@ func getPatterns(args []string) ([]glob.Glob, error) {
 	}
 	pat := make([]glob.Glob, len(args))
 	for i := range args {
-		if g, err := glob.Compile(args[i], '/'); err != nil {
+		g, err := glob.Compile(args[i], '/')
+		if err != nil {
 			return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err)
-		} else {
-			pat[i] = g
 		}
+		pat[i] = g
 	}
 	return pat, nil
 }

cmd/embedded.go

@@ -1,7 +1,7 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-//go:build !bindata
+//go:build servedynamic
 
 package cmd
 
@@ -23,7 +23,7 @@ var (
 )
 
 func extractorNotImplemented(c *cli.Context) error {
-	err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata")
+	err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in servedynamic builds")
 	fmt.Fprintf(os.Stderr, "%s\n", err)
 	return err
 }

cmd/embedded_stub.go

@@ -81,7 +81,7 @@ One of these three distributions of Make will run on Windows:
   - To compile Gitea with CGO_ENABLED (eg: SQLite3), you might need to use [tdm-gcc](https://jmeubank.github.io/tdm-gcc/) instead of MSYS2 gcc, because MSYS2 gcc headers lack some Windows-only CRT functions like `_beginthread`.
 - [Chocolatey package](https://chocolatey.org/packages/make). Run `choco install make`
 
-**Note**: If you are attempting to build using make with Windows Command Prompt, you may run into issues. The above prompts (Git bash, or MinGW) are recommended, however if you only have command prompt (or potentially PowerShell) you can set environment variables using the [set](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1) command, e.g. `set TAGS=bindata`.
+**Note**: If you are attempting to build using make with Windows Command Prompt, you may run into issues. The above prompts (Git bash, or MinGW) are recommended, however if you only have command prompt (or potentially PowerShell) you can set environment variables using the [set](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1) command, e.g. `set TAGS=servedynamic`.
 
 ## Downloading and cloning the Gitea source code
 
@@ -128,10 +128,10 @@ from source</a>.
 The simplest recommended way to build from source is:
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make build
+TAGS="sqlite sqlite_unlock_notify" make build
 ```
 
-The `build` target will execute both `frontend` and `backend` sub-targets. If the `bindata` tag is present, the frontend files will be compiled into the binary. It is recommended to leave out the tag when doing frontend development so that changes will be reflected.
+The `build` target will execute both `frontend` and `backend` sub-targets. If the `servedynamic` tag is present, the frontend files will not be compiled into the binary. It is recommended to set `servedynamic` tag when doing frontend development so that changes will be reflected.
 
 See `make help` for all available `make` targets. Also see [`.drone.yml`](https://github.com/go-gitea/gitea/blob/main/.drone.yml) to see how our continuous integration works.
 
@@ -299,7 +299,7 @@ Unit tests are covered by `*_test.go` in `go test` system.
 You can set the environment variable `GITEA_UNIT_TESTS_LOG_SQL=1` to display all SQL statements when running the tests in verbose mode (i.e. when `GOTESTFLAGS=-v` is set).
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make test # Runs the unit tests
+TAGS="sqlite sqlite_unlock_notify" make test # Runs the unit tests
 ```
 
 ### Integration Tests
@@ -308,7 +308,7 @@ Unit tests will not and cannot completely test Gitea alone. Therefore, we
 have written integration tests; however, these are database dependent.
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make build test-sqlite
+TAGS="sqlite sqlite_unlock_notify" make build test-sqlite
 ```
 
 will run the integration tests in an SQLite environment. Integration tests

docs/content/doc/developers/hacking-on-gitea.en-us.md

@@ -94,7 +94,7 @@ are provided to keep the build process as simple as possible.
 
 Depending on requirements, the following build tags can be included.
 
-- `bindata`: Build a single monolithic binary, with all assets included. Required for production build.
+- `servedynamic`: Read files from the filesystem instead of embedded.
 - `sqlite sqlite_unlock_notify`: Enable support for a
   [SQLite3](https://sqlite.org/) database. Suggested only for tiny
   installations.
@@ -103,21 +103,19 @@ Depending on requirements, the following build tags can be included.
   available to PAM.
 - `gogit`: (EXPERIMENTAL) Use go-git variants of Git commands.
 
-Bundling all assets (JS/CSS/templates, etc) into the binary. Using the `bindata` build tag is required for
-production deployments. You could exclude `bindata` when you are developing/testing Gitea or able to separate the assets correctly.
-
-To include all assets, use the `bindata` tag:
+You could use tag `servedynamic` when you are developing/testing Gitea to serve
+assets from the filesystem instead of the embedded versions.
 
 ```bash
-TAGS="bindata" make build
+make build
 ```
 
 In the default release build of our continuous integration system, the build
-tags are: `TAGS="bindata sqlite sqlite_unlock_notify"`. The simplest
+tags are: `TAGS="sqlite sqlite_unlock_notify"`. The simplest
 recommended way to build from source is therefore:
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make build
+TAGS="sqlite sqlite_unlock_notify" make build
 ```
 
 The `build` target is split into two sub-targets:
@@ -128,7 +126,7 @@ The `build` target is split into two sub-targets:
 If pre-built frontend files are present it is possible to only build the backend:
 
 ```bash
-TAGS="bindata" make backend
+make backend
 ```
 
 ## Test
@@ -181,7 +179,7 @@ GOOS=linux GOARCH=arm64 make build
 Cross-build Gitea for Linux ARM64, with recommended build tags:
 
 ```
-CC=aarch64-unknown-linux-gnu-gcc GOOS=linux GOARCH=arm64 TAGS="bindata sqlite sqlite_unlock_notify" make build
+CC=aarch64-unknown-linux-gnu-gcc GOOS=linux GOARCH=arm64 TAGS="sqlite sqlite_unlock_notify" make build
 ```
 
 Replace `CC`, `GOOS`, and `GOARCH` as appropriate for your architecture target.

docs/content/doc/installation/from-source.en-us.md

@@ -28,7 +28,7 @@ menu:
 All global options can be placed at the command level.
 
 - `--help`, `-h`: Show help text and exit. Optional.
-- `--version`, `-v`: Show version and exit. Optional. (example: `Gitea version 1.1.0+218-g7b907ed built with: bindata, sqlite`).
+- `--version`, `-v`: Show version and exit. Optional. (example: `Gitea version 1.1.0+218-g7b907ed built with: sqlite`).
 - `--custom-path path`, `-C path`: Location of the Gitea custom folder. Optional. (default: `AppWorkPath`/custom or `$GITEA_CUSTOM`).
 - `--config path`, `-c path`: Gitea configuration file path. Optional. (default: `custom`/conf/app.ini).
 - `--work-path path`, `-w path`: Gitea `AppWorkPath`. Optional. (default: LOCATION_OF_GITEA_BINARY or `$GITEA_WORK_DIR`)