Merge pull request #128 from WJQSERVER-STUDIO/main

sync

CHANGELOG.md

@@ -1,5 +1,54 @@
 # 更新日志
 
+3.5.6 - 2025-06-15
+---
+- FIX: 修正blob重写的生成问题
+- CHANGE: 改进302重定向逻辑
+
+25w48c - 2025-06-15
+---
+- PRE-RELEASE: 此版本是v3.5.6预发布版本,请勿在生产环境中使用;
+- CHANGE: 加入内部301处理
+
+25w48b - 2025-06-15
+---
+- PRE-RELEASE: 此版本是v3.5.6预发布版本,请勿在生产环境中使用;
+- FIX: 修正blob重写的生成问题
+- CHANGE: 验证与连接释放相关的修正
+
+25w48a - 2025-06-14
+---
+- PRE-RELEASE: 此版本是v3.5.6预发布版本,请勿在生产环境中使用;
+- CHANGE: 测试302重定向逻辑
+
+3.5.5 - 2025-06-14
+---
+- CHANGE: 修正新匹配器的覆盖问题, 同时增加test的覆盖
+
+25w47a - 2025-06-14
+---
+- PRE-RELEASE: 此版本是v3.5.5预发布版本,请勿在生产环境中使用;
+- CHANGE: 修正新匹配器的覆盖问题, 同时增加test的覆盖
+
+3.5.4 - 2025-06-14
+---
+- CHANGE: 移植来自于[GHProxy-Touka](https://github.com/WJQSERVER-STUDIO/ghproxy-touka)的blob处理逻辑与302处理逻辑
+
+25w46c - 2025-06-14
+---
+- PRE-RELEASE: 此版本是v3.5.4预发布版本,请勿在生产环境中使用;
+- CHANGE: 移植来自于[GHProxy-Touka](https://github.com/WJQSERVER-STUDIO/ghproxy-touka)的blob处理逻辑与302处理逻辑
+
+25w46b - 2025-06-14
+---
+- PRE-RELEASE: 此版本是v3.5.4预发布版本,请勿在生产环境中使用;
+- CHANGE: 修改关闭行为以测试问题
+
+25w46a - 2025-06-14
+---
+- PRE-RELEASE: 此版本是v3.5.4预发布版本,请勿在生产环境中使用;
+- CHANGE: 修改payload行为以测试问题
+
 3.5.3 - 2025-06-13
 ---
 - CHANGE: 显式配置`WithStreamBody(true)`

DEV-VERSION

@@ -1 +1 @@
-25w45a
+25w48c

README.md

@@ -109,7 +109,7 @@ wget -O install-dev.sh https://raw.githubusercontent.com/WJQSERVER-STUDIO/ghprox
 
 ## LICENSE
 
-v3.5.2开始, 本项目使用 [WJQserver Studio License 2.0](https://wjqserver-studio.github.io/LICENSE/LICENSE.html) 和 [Mozilla Public License Version 2.0](https://mozilla.org/MPL/2.0/) 双重许可, 您可从中选择一个使用
+v3.5.2开始, 本项目使用 [WJQserver Studio License 2.1](https://wjqserver-studio.github.io/LICENSE/LICENSE.html) 和 [Mozilla Public License Version 2.0](https://mozilla.org/MPL/2.0/) 双重许可, 您可从中选择一个使用
 
 前端位于单独仓库中, 且各个主题均存在各自的许可证, 本项目许可证并不包括前端
 

VERSION

@@ -1 +1 @@
-3.5.3
+3.5.6

config/config.go

@@ -26,6 +26,7 @@ type Config struct {
 host = "0.0.0.0"
 port = 8080
 netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net"
+goPoolSize = 1024
 sizeLimit = 125 # MB
 memLimit = 0 # MB
 H2C = true
@@ -38,6 +39,7 @@ type ServerConfig struct {
 	Host                     string `toml:"host"`
 	NetLib                   string `toml:"netlib"`
 	SenseClientDisconnection bool   `toml:"senseClientDisconnection"`
+	GoPoolSize               int    `toml:"goPoolSize"`
 	SizeLimit                int    `toml:"sizeLimit"`
 	MemLimit                 int64  `toml:"memLimit"`
 	H2C                      bool   `toml:"H2C"`
@@ -224,14 +226,15 @@ func FileExists(filename string) bool {
 func DefaultConfig() *Config {
 	return &Config{
 		Server: ServerConfig{
-			Port:      8080,
-			Host:      "0.0.0.0",
-			NetLib:    "netpoll",
-			SizeLimit: 125,
-			MemLimit:  0,
-			H2C:       true,
-			Cors:      "*",
-			Debug:     false,
+			Port:       8080,
+			Host:       "0.0.0.0",
+			NetLib:     "netpoll",
+			GoPoolSize: 1024,
+			SizeLimit:  125,
+			MemLimit:   0,
+			H2C:        true,
+			Cors:       "*",
+			Debug:      false,
 		},
 		Httpc: HttpcConfig{
 			Mode:                "auto",

config/config.toml

@@ -3,6 +3,7 @@ host = "0.0.0.0"
 port = 8080 
 netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net"
 senseClientDisconnection = false
+goPoolSize = 1024
 sizeLimit = 125 # MB
 memLimit = 0 # MB
 H2C = true

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/BurntSushi/toml v1.5.0
 	github.com/WJQSERVER-STUDIO/httpc v0.7.0
 	github.com/WJQSERVER-STUDIO/logger v1.8.0
-	github.com/cloudwego/hertz v0.10.0
+	github.com/cloudwego/hertz v0.10.1-0.20250611091639-3dde619f5598
 	github.com/hertz-contrib/http2 v0.1.8
 	golang.org/x/net v0.41.0
 	golang.org/x/time v0.12.0
@@ -44,6 +44,3 @@ require (
 )
 
 replace github.com/nyaruka/phonenumbers => github.com/nyaruka/phonenumbers v1.6.1 // 1.6.3 has reflect leaking
-
-//replace github.com/WJQSERVER-STUDIO/httpc v0.5.1 => /data/github/WJQSERVER-STUDIO/httpc
-//replace github.com/WJQSERVER-STUDIO/logger v1.6.0 => /data/github/WJQSERVER-STUDIO/logger

go.sum

@@ -24,8 +24,8 @@ github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCy
 github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/gopkg v0.1.4 h1:EoQiCG4sTonTPHxOGE0VlQs+sQR+Hsi2uN0qqwu8O50=
 github.com/cloudwego/gopkg v0.1.4/go.mod h1:FQuXsRWRsSqJLsMVd5SYzp8/Z1y5gXKnVvRrWUOsCMI=
-github.com/cloudwego/hertz v0.10.0 h1:V0vmBaLdQPlgL6w2TA6PZL1g6SGgQznFx6vqxWdCcKw=
-github.com/cloudwego/hertz v0.10.0/go.mod h1:lRBohmcDkGx5TLK6QKFGdzJ6n3IXqGueHsOiXcYgXA4=
+github.com/cloudwego/hertz v0.10.1-0.20250611091639-3dde619f5598 h1:8tVol3hNJS7+7f7yQIkP57tZMzUV3fOhn6pQ7t4R06k=
+github.com/cloudwego/hertz v0.10.1-0.20250611091639-3dde619f5598/go.mod h1:lRBohmcDkGx5TLK6QKFGdzJ6n3IXqGueHsOiXcYgXA4=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/cloudwego/netpoll v0.7.0 h1:bDrxQaNfijRI1zyGgXHQoE/nYegL0nr+ijO1Norelc4=
 github.com/cloudwego/netpoll v0.7.0/go.mod h1:PI+YrmyS7cIr0+SD4seJz3Eo3ckkXdu2ZVKBLhURLNU=

main.go

@@ -431,6 +431,7 @@ func main() {
 				server.WithHostPorts(addr),
 				server.WithTransport(standard.NewTransporter),
 				server.WithStreamBody(true),
+				server.WithIdleTimeout(30*time.Second),
 			)
 			r.AddProtocol("h2", factory.NewServerFactory())
 		} else {
@@ -438,6 +439,7 @@ func main() {
 				server.WithHostPorts(addr),
 				server.WithTransport(standard.NewTransporter),
 				server.WithStreamBody(true),
+				server.WithIdleTimeout(30*time.Second),
 			)
 		}
 	} else if cfg.Server.NetLib == "netpoll" || cfg.Server.NetLib == "" {
@@ -447,6 +449,7 @@ func main() {
 				server.WithHostPorts(addr),
 				server.WithSenseClientDisconnection(cfg.Server.SenseClientDisconnection),
 				server.WithStreamBody(true),
+				server.WithIdleTimeout(30*time.Second),
 			)
 			r.AddProtocol("h2", factory.NewServerFactory())
 		} else {
@@ -454,6 +457,7 @@ func main() {
 				server.WithHostPorts(addr),
 				server.WithSenseClientDisconnection(cfg.Server.SenseClientDisconnection),
 				server.WithStreamBody(true),
+				server.WithIdleTimeout(30*time.Second),
 			)
 		}
 	} else {
@@ -462,6 +466,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	/*
+		if cfg.Server.GoPoolSize > 0 {
+			gopool.SetCap(int32(cfg.Server.GoPoolSize))
+		} else {
+			gopool.SetCap(1024)
+		}
+	*/
+
 	r.Use(recovery.Recovery()) // Recovery中间件
 	r.Use(loggin.Middleware()) // log中间件
 	r.Use(viaHeader())
@@ -507,7 +519,7 @@ func main() {
 		proxy.NoRouteHandler(cfg, limiter, iplimiter)(ctx, c)
 	})
 
-	r.GET("/api.github.com/repos/:user/:repo/*filepath", func(ctx context.Context, c *app.RequestContext) {
+	r.Any("/api.github.com/repos/:user/:repo/*filepath", func(ctx context.Context, c *app.RequestContext) {
 		c.Set("matcher", "api")
 		proxy.RoutingHandler(cfg, limiter, iplimiter)(ctx, c)
 	})

proxy/chunkreq.go

@@ -23,16 +23,18 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c
 	go func() {
 		<-ctx.Done()
 		if resp != nil && resp.Body != nil {
-			resp.Body.Close()
-		}
-		if req != nil {
-			req.Body.Close()
+			err := resp.Body.Close()
+			if err != nil {
+				logError("Failed to close response body: %v", err)
+			}
 		}
+		c.Abort()
 	}()
 
 	rb := client.NewRequestBuilder(string(c.Request.Method()), u)
 	rb.NoDefaultHeaders()
-	rb.SetBody(c.Request.BodyStream())
+	//rb.SetBody(bytes.NewBuffer(c.Request.Body()))
+	rb.SetBody(c.RequestBodyStream())
 	rb.WithContext(ctx)
 
 	req, err = rb.Build()
@@ -56,6 +58,21 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c
 		return
 	}
 
+	// 处理302情况
+	if resp.StatusCode == 302 || resp.StatusCode == 301 {
+		finalURL := resp.Header.Get("Location")
+		if finalURL != "" {
+			err = resp.Body.Close()
+			if err != nil {
+				logError("Failed to close response body: %v", err)
+			}
+			c.Request.Header.Del("Referer")
+			logInfo("Internal Redirecting to %s", finalURL)
+			ChunkedProxyRequest(ctx, c, finalURL, cfg, matcher)
+			return
+		}
+	}
+
 	var (
 		bodySize      int
 		contentLength string
@@ -110,8 +127,6 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c
 		bodyReader = limitreader.NewRateLimitedReader(bodyReader, bandwidthLimit, int(bandwidthBurst), ctx)
 	}
 
-	defer bodyReader.Close()
-
 	if MatcherShell(u) && matchString(matcher) && cfg.Shell.Editor {
 		// 判断body是不是gzip
 		var compress string
@@ -138,6 +153,7 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c
 			return
 		}
 		c.SetBodyStream(bodyReader, -1)
+		bodyReader.Close()
 	}
 
 }

proxy/gitreq.go

@@ -17,15 +17,16 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 	var (
 		req  *http.Request
 		resp *http.Response
+		err  error
 	)
 
 	go func() {
 		<-ctx.Done()
 		if resp != nil && resp.Body != nil {
-			resp.Body.Close()
-		}
-		if req != nil {
-			req.Body.Close()
+			err = resp.Body.Close()
+			if err != nil {
+				logError("Failed to close response body: %v", err)
+			}
 		}
 	}()
 
@@ -51,7 +52,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 		rb.SetBody(reqBodyReader)
 		rb.WithContext(ctx)
 
-		req, err := rb.Build()
+		req, err = rb.Build()
 		if err != nil {
 			HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
 			return
@@ -143,4 +144,5 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 	}
 
 	c.SetBodyStream(bodyReader, -1)
+	bodyReader.Close()
 }

proxy/handler.go

@@ -68,7 +68,10 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra
 
 		// 处理blob/raw路径
 		if matcher == "blob" {
-			rawPath = strings.Replace(rawPath, "/blob/", "/raw/", 1)
+			rawPath = rawPath[18:]
+			rawPath = "https://raw.githubusercontent.com" + rawPath
+			rawPath = strings.Replace(rawPath, "/blob/", "/", 1)
+			matcher = "raw"
 		}
 
 		logDebug("Matched: %v", matcher)

proxy/match.go

@@ -10,14 +10,16 @@ import (
 )
 
 var (
-	githubPrefix    = "https://github.com/"
-	rawPrefix       = "https://raw.githubusercontent.com/"
-	gistPrefix      = "https://gist.github.com/"
-	apiPrefix       = "https://api.github.com/"
-	githubPrefixLen int
-	rawPrefixLen    int
-	gistPrefixLen   int
-	apiPrefixLen    int
+	githubPrefix         = "https://github.com/"
+	rawPrefix            = "https://raw.githubusercontent.com/"
+	gistPrefix           = "https://gist.github.com/"
+	gistContentPrefix    = "https://gist.githubusercontent.com/"
+	apiPrefix            = "https://api.github.com/"
+	githubPrefixLen      int
+	rawPrefixLen         int
+	gistPrefixLen        int
+	gistContentPrefixLen int
+	apiPrefixLen         int
 )
 
 func init() {
@@ -25,6 +27,7 @@ func init() {
 	rawPrefixLen = len(rawPrefix)
 	gistPrefixLen = len(gistPrefix)
 	apiPrefixLen = len(apiPrefix)
+	gistContentPrefixLen = len(gistContentPrefix)
 	//log.Printf("githubPrefixLen: %d, rawPrefixLen: %d, gistPrefixLen: %d, apiPrefixLen: %d", githubPrefixLen, rawPrefixLen, gistPrefixLen, apiPrefixLen)
 }
 
@@ -114,6 +117,23 @@ func Matcher(rawPath string, cfg *config.Config) (string, string, string, *GHPro
 		return user, "", "gist", nil
 	}
 
+	// 匹配 "https://gist.githubusercontent.com/"
+	if strings.HasPrefix(rawPath, gistContentPrefix) {
+		remaining := rawPath[gistContentPrefixLen:]
+		i := strings.IndexByte(remaining, '/')
+		if i <= 0 {
+			// case: https://gist.githubusercontent.com/user
+			// 这种情况下, gist_id 缺失, 但我们仍然可以认为 user 是有效的
+			if len(remaining) > 0 {
+				return remaining, "", "gist", nil
+			}
+			return "", "", "", NewErrorWithStatusLookup(400, "malformed gist url: missing user")
+		}
+		// case: https://gist.githubusercontent.com/user/gist_id...
+		user := remaining[:i]
+		return user, "", "gist", nil
+	}
+
 	// 匹配 "https://api.github.com/"
 	if strings.HasPrefix(rawPath, apiPrefix) {
 		if !cfg.Auth.ForceAllowApi && (cfg.Auth.Method != "header" || !cfg.Auth.Enabled) {

proxy/matcher_test.go

@@ -87,6 +87,12 @@ func TestMatcher_Compatibility(t *testing.T) {
 			config:       cfgWithAuth,
 			expectedUser: "user", expectedRepo: "", expectedMatcher: "gist",
 		},
+		{
+			name:         "Gist UserContent Path",
+			rawPath:      "https://gist.githubusercontent.com/user/abcdef1234567890",
+			config:       cfgWithAuth,
+			expectedUser: "user", expectedRepo: "", expectedMatcher: "gist",
+		},
 		{
 			name:         "API Repos Path (with Auth)",
 			rawPath:      "https://api.github.com/repos/owner/repo/pulls",

proxy/reqheader.go

@@ -49,6 +49,16 @@ var (
 	}
 )
 
+// copyHeader 将所有头部从 src 复制到 dst。
+// 对于多值头部，它会为每个值调用 Add，从而保留所有值。
+func copyHeader(dst, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}
+
 func setRequestHeaders(c *app.RequestContext, req *http.Request, cfg *config.Config, matcher string) {
 	if matcher == "raw" && cfg.Httpc.UseCustomRawHeaders {
 		// 使用预定义Header
@@ -56,20 +66,23 @@ func setRequestHeaders(c *app.RequestContext, req *http.Request, cfg *config.Con
 			req.Header.Set(key, value)
 		}
 	} else if matcher == "clone" {
+
 		c.Request.Header.VisitAll(func(key, value []byte) {
 			headerKey := string(key)
 			headerValue := string(value)
-			if _, shouldRemove := cloneHeadersToRemove[headerKey]; !shouldRemove {
-				req.Header.Set(headerKey, headerValue)
-			}
+			req.Header.Set(headerKey, headerValue)
 		})
+		for key := range cloneHeadersToRemove {
+			req.Header.Del(key)
+		}
 	} else {
 		c.Request.Header.VisitAll(func(key, value []byte) {
 			headerKey := string(key)
 			headerValue := string(value)
-			if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove {
-				req.Header.Set(headerKey, headerValue)
-			}
+			req.Header.Set(headerKey, headerValue)
 		})
+		for key := range reqHeadersToRemove {
+			req.Header.Del(key)
+		}
 	}
 }

proxy/routing.go

@@ -48,9 +48,13 @@ func RoutingHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra
 			return
 		}
 
+		// 处理blob/raw路径
 		// 处理blob/raw路径
 		if matcher == "blob" {
-			rawPath = strings.Replace(rawPath, "/blob/", "/raw/", 1)
+			rawPath = rawPath[10:]
+			rawPath = "raw.githubusercontent.com" + rawPath
+			rawPath = strings.Replace(rawPath, "/blob/", "/", 1)
+			matcher = "raw"
 		}
 
 		// 为rawpath加入https:// 头