Support system proxy

lunny · lunny · commit 5e5a764588bc · 2021-04-17T14:36:57.000+08:00
diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
@@ -1302,3 +1302,11 @@ STORAGE_TYPE = local
 ;MINIO_LOCATION = us-east-1
 ; Minio enabled ssl only available when STORAGE_TYPE is `minio`
 ;MINIO_USE_SSL = false
+
+[proxy]
+; Enable the proxy, all requests to external via HTTP will be affected
+PROXY_ENABLED = false
+; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
+PROXY_URL =
+; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+PROXY_HOSTS =
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -523,8 +523,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
 - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
 - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
 - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
-- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
-- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting.
+- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting.
 
 ## Mailer (`mailer`)
 
@@ -902,6 +902,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
 - `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
+- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
 
 ## Mirror (`mirror`)
 
@@ -958,6 +959,19 @@ MINIO_USE_SSL = false
 
 And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`.
 
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **true**: Enable the proxy, all requests to external via HTTP will be affected
+- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
+- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
@@ -326,7 +326,8 @@ IS_INPUT_FILE = false
 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。
 - `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。
 - `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。
-- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
+- `ALLOW_LOCALNETWORKS`: **false**: 允许访问私有地址，定义在 RFC 1918
+- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
 
 ## LFS (`lfs`)
 
@@ -377,6 +378,19 @@ MINIO_USE_SSL = false
 
 然后你在 `[attachment]`, `[lfs]` 等中可以把这个名字用作 `STORAGE_TYPE` 的值。
 
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **true**: 是否启用全局代理
+- `PROXY_URL`: ****: 代理服务器地址，支持 http://, https//, socks://，为空则不启用代理而使用环境变量中的 http_proxy/https_proxy
+- `PROXY_HOSTS`: ****: 逗号分隔的多个需要代理的网址，支持 * 号匹配符号， ** 表示匹配所有网站
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,8 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	gitea_sdk "code.gitea.io/sdk/gitea"
@@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
 		gitea_sdk.SetToken(token),
 		gitea_sdk.SetBasicAuth(username, password),
 		gitea_sdk.SetContext(ctx),
+		gitea_sdk.SetHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+				Proxy:           proxy.SystemProxy(),
+			},
+		}),
 	)
 	if err != nil {
 		log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
@@ -89,7 +90,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
 		Transport: &http.Transport{
 			Proxy: func(req *http.Request) (*url.URL, error) {
 				req.SetBasicAuth(userName, password)
-				return nil, nil
+				return proxy.SystemProxy()(req)
 			},
 		},
 	}
diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,8 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/xanzy/go-gitlab"
@@ -77,7 +80,12 @@ type GitlabDownloader struct {
 //   Use either a username/password, personal token entered into the username field, or anonymous/public access
 //   Note: Public access only allows very basic access
 func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
-	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
+	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+			Proxy:           proxy.SystemProxy(),
+		},
+	}))
 	// Only use basic auth if token is blank and password is NOT
 	// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
 	if token == "" && password != "" {
diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -14,6 +15,8 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/gogs/go-gogs-client"
@@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
 		downloader.userName = token
 	} else {
 		downloader.transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
 			Proxy: func(req *http.Request) (*url.URL, error) {
 				req.SetBasicAuth(userName, password)
-				return nil, nil
+				return proxy.SystemProxy()(req)
 			},
 		}
 
diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go
@@ -0,0 +1,47 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+	"net/http"
+	"net/url"
+	"sync"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/gobwas/glob"
+)
+
+var (
+	once         sync.Once
+	hostMatchers []glob.Glob
+)
+
+// SystemProxy returns the system proxy
+func SystemProxy() func(req *http.Request) (*url.URL, error) {
+	if !setting.Proxy.Enabled || setting.Proxy.ProxyURL == "" {
+		return http.ProxyFromEnvironment
+	}
+
+	once.Do(func() {
+		for _, h := range setting.Proxy.ProxyHosts {
+			if g, err := glob.Compile(h); err == nil {
+				hostMatchers = append(hostMatchers, g)
+			} else {
+				log.Error("glob.Compile %s failed: %v", h, err)
+			}
+		}
+	})
+
+	return func(req *http.Request) (*url.URL, error) {
+		for _, v := range hostMatchers {
+			if v.Match(req.URL.Host) {
+				return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
+			}
+		}
+		return http.ProxyFromEnvironment(req)
+	}
+}
diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go
@@ -16,6 +16,7 @@ var (
 		AllowedDomains     []string
 		BlockedDomains     []string
 		AllowLocalNetworks bool
+		SkipTLSVerify      bool
 	}{
 		MaxAttempts:  3,
 		RetryBackoff: 3,
@@ -37,4 +38,5 @@ func newMigrationsService() {
 	}
 
 	Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
+	Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
 }
diff --git a/modules/setting/proxy.go b/modules/setting/proxy.go
@@ -0,0 +1,40 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"net/url"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+var (
+	// Proxy settings
+	Proxy = struct {
+		Enabled       bool
+		ProxyURL      string
+		ProxyURLFixed *url.URL
+		ProxyHosts    []string
+	}{
+		Enabled:    false,
+		ProxyURL:   "",
+		ProxyHosts: []string{},
+	}
+)
+
+func newProxyService() {
+	sec := Cfg.Section("proxy")
+	Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
+	Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
+	if Proxy.ProxyURL != "" {
+		var err error
+		Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
+		if err != nil {
+			log.Error("Global PROXY_URL is not valid")
+			Proxy.ProxyURL = ""
+		}
+	}
+	Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
@@ -1171,6 +1171,7 @@ func NewServices() {
 	newMailService()
 	newRegisterMailService()
 	newNotifyMailService()
+	newProxyService()
 	newWebhookService()
 	newMigrationsService()
 	newIndexerService()
diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go
@@ -20,7 +20,9 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/proxy"
 	"code.gitea.io/gitea/modules/setting"
+
 	"github.com/gobwas/glob"
 )
 
@@ -239,7 +241,7 @@ var (
 
 func webhookProxy() func(req *http.Request) (*url.URL, error) {
 	if setting.Webhook.ProxyURL == "" {
-		return http.ProxyFromEnvironment
+		return proxy.SystemProxy()
 	}
 
 	once.Do(func() {

Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,7 @@ var (`
`16`	`16`	`AllowedDomains []string`
`17`	`17`	`BlockedDomains []string`
`18`	`18`	`AllowLocalNetworks bool`
	`19`	`+ SkipTLSVerify bool`
`19`	`20`	`}{`
`20`	`21`	`MaxAttempts: 3,`
`21`	`22`	`RetryBackoff: 3,`
`@@ -37,4 +38,5 @@ func newMigrationsService() {`
`37`	`38`	`}`
`38`	`39`
`39`	`40`	`Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)`
	`41`	`+ Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)`
`40`	`42`	`}`