add support for archive-upload rpc (#36391)

Add support for fetching archives with `git archive --remote <repo-url>`

closes: https://github.com/go-gitea/gitea/issues/23425

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

routers/web/githttp.go

@@ -10,9 +10,12 @@ import (
 )
 
 func addOwnerRepoGitHTTPRouters(m *web.Router) {
+	// Some users want to use "web-based git client" to access Gitea's repositories,
+	// so the CORS handler and OPTIONS method are used.
 	m.Group("/{username}/{reponame}", func() {
 		m.Methods("POST,OPTIONS", "/git-upload-pack", repo.ServiceUploadPack)
 		m.Methods("POST,OPTIONS", "/git-receive-pack", repo.ServiceReceivePack)
+		m.Methods("POST,OPTIONS", "/git-upload-archive", repo.ServiceUploadArchive)
 		m.Methods("GET,OPTIONS", "/info/refs", repo.GetInfoRefs)
 		m.Methods("GET,OPTIONS", "/HEAD", repo.GetTextFile("HEAD"))
 		m.Methods("GET,OPTIONS", "/objects/info/alternates", repo.GetTextFile("objects/info/alternates"))

routers/web/repo/githttp.go

@@ -30,6 +30,7 @@ import (
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/services/context"
 	repo_service "code.gitea.io/gitea/services/repository"
 
@@ -55,8 +56,9 @@ func CorsHandler() func(next http.Handler) http.Handler {
 	}
 }
 
-// httpBase implementation git smart HTTP protocol
+// httpBase does the common work for git http services,
-func httpBase(ctx *context.Context) *serviceHandler {
+// including early response, authentication, repository lookup and permission check.
+func httpBase(ctx *context.Context, optGitService ...string) *serviceHandler {
 	username := ctx.PathParam("username")
 	reponame := strings.TrimSuffix(ctx.PathParam("reponame"), ".git")
 
@@ -65,20 +67,23 @@ func httpBase(ctx *context.Context) *serviceHandler {
 		return nil
 	}
 
+	var serviceType string
 	var isPull, receivePack bool
-	service := ctx.FormString("service")
+	switch util.OptionalArg(optGitService) {
-	if service == "git-receive-pack" ||
+	case "git-receive-pack":
-		strings.HasSuffix(ctx.Req.URL.Path, "git-receive-pack") {
+		serviceType = ServiceTypeReceivePack
-		isPull = false
 		receivePack = true
-	} else if service == "git-upload-pack" ||
+	case "git-upload-pack":
-		strings.HasSuffix(ctx.Req.URL.Path, "git-upload-pack") {
+		serviceType = ServiceTypeUploadPack
 		isPull = true
-	} else if service == "git-upload-archive" ||
+	case "git-upload-archive":
-		strings.HasSuffix(ctx.Req.URL.Path, "git-upload-archive") {
+		serviceType = ServiceTypeUploadArchive
 		isPull = true
-	} else {
+	case "":
 		isPull = ctx.Req.Method == http.MethodHead || ctx.Req.Method == http.MethodGet
+	default: // unknown service
+		ctx.Resp.WriteHeader(http.StatusBadRequest)
+		return nil
 	}
 
 	var accessMode perm.AccessMode
@@ -188,7 +193,7 @@ func httpBase(ctx *context.Context) *serviceHandler {
 		}
 
 		if repoExist {
-			// Because of special ref "refs/for" .. , need delay write permission check
+			// Because of special ref "refs/for" (agit) , need delay write permission check
 			if git.DefaultFeatures().SupportProcReceive {
 				accessMode = perm.AccessModeRead
 			}
@@ -277,7 +282,6 @@ func httpBase(ctx *context.Context) *serviceHandler {
 				ctx.PlainText(http.StatusForbidden, "repository wiki is disabled")
 				return nil
 			}
-			log.Error("Failed to get the wiki unit in %-v Error: %v", repo, err)
 			ctx.ServerError("GetUnit(UnitTypeWiki) for "+repo.FullName(), err)
 			return nil
 		}
@@ -285,9 +289,7 @@ func httpBase(ctx *context.Context) *serviceHandler {
 
 	environ = append(environ, repo_module.EnvRepoID+fmt.Sprintf("=%d", repo.ID))
 
-	ctx.Req.URL.Path = strings.ToLower(ctx.Req.URL.Path) // blue: In case some repo name has upper case name
+	return &serviceHandler{serviceType, repo, isWiki, environ}
-
-	return &serviceHandler{repo, isWiki, environ}
 }
 
 var (
@@ -330,6 +332,8 @@ func dummyInfoRefs(ctx *context.Context) {
 }
 
 type serviceHandler struct {
+	serviceType string
+
 	repo    *repo_model.Repository
 	isWiki  bool
 	environ []string
@@ -350,7 +354,7 @@ func setHeaderNoCache(ctx *context.Context) {
 
 func setHeaderCacheForever(ctx *context.Context) {
 	now := time.Now().Unix()
-	expires := now + 31536000
+	expires := now + 365*86400 // 365 days
 	ctx.Resp.Header().Set("Date", strconv.FormatInt(now, 10))
 	ctx.Resp.Header().Set("Expires", strconv.FormatInt(expires, 10))
 	ctx.Resp.Header().Set("Cache-Control", "public, max-age=31536000")
@@ -367,7 +371,7 @@ func isSlashRune(r rune) bool { return r == '/' || r == '\\' }
 
 func (h *serviceHandler) sendFile(ctx *context.Context, contentType, file string) {
 	if containsParentDirectorySeparator(file) {
-		log.Error("request file path contains invalid path: %v", file)
+		log.Debug("request file path contains invalid path: %v", file)
 		ctx.Resp.WriteHeader(http.StatusBadRequest)
 		return
 	}
@@ -380,38 +384,45 @@ func (h *serviceHandler) sendFile(ctx *context.Context, contentType, file string
 // one or more key=value pairs separated by colons
 var safeGitProtocolHeader = regexp.MustCompile(`^[0-9a-zA-Z]+=[0-9a-zA-Z]+(:[0-9a-zA-Z]+=[0-9a-zA-Z]+)*$`)
 
-func prepareGitCmdWithAllowedService(service string) (*gitcmd.Command, error) {
+func prepareGitCmdWithAllowedService(service string, allowedServices []string) *gitcmd.Command {
-	if service == ServiceTypeReceivePack {
+	if !slices.Contains(allowedServices, service) {
-		return gitcmd.NewCommand(ServiceTypeReceivePack), nil
+		return nil
 	}
-	if service == ServiceTypeUploadPack {
+	switch service {
-		return gitcmd.NewCommand(ServiceTypeUploadPack), nil
+	case ServiceTypeReceivePack:
+		return gitcmd.NewCommand(ServiceTypeReceivePack)
+	case ServiceTypeUploadPack:
+		return gitcmd.NewCommand(ServiceTypeUploadPack)
+	case ServiceTypeUploadArchive:
+		return gitcmd.NewCommand(ServiceTypeUploadArchive)
+	default:
+		return nil
 	}
-	return nil, fmt.Errorf("service %q is not allowed", service)
 }
 
-func serviceRPC(ctx *context.Context, h *serviceHandler, service string) {
+func serviceRPC(ctx *context.Context, service string) {
-	defer func() {
+	defer ctx.Req.Body.Close()
-		if err := ctx.Req.Body.Close(); err != nil {
+	h := httpBase(ctx, "git-"+service)
-			log.Error("serviceRPC: Close: %v", err)
+	if h == nil {
-		}
+		return
-	}()
+	}
 
 	expectedContentType := fmt.Sprintf("application/x-git-%s-request", service)
 	if ctx.Req.Header.Get("Content-Type") != expectedContentType {
-		log.Error("Content-Type (%q) doesn't match expected: %q", ctx.Req.Header.Get("Content-Type"), expectedContentType)
+		log.Debug("Content-Type (%q) doesn't match expected: %q", ctx.Req.Header.Get("Content-Type"), expectedContentType)
-		// FIXME: why it's 401 if the content type is unexpected?
+		ctx.Resp.WriteHeader(http.StatusBadRequest)
-		ctx.Resp.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
-	cmd, err := prepareGitCmdWithAllowedService(service)
+	cmd := prepareGitCmdWithAllowedService(service, []string{ServiceTypeUploadPack, ServiceTypeReceivePack, ServiceTypeUploadArchive})
-	if err != nil {
+	if cmd == nil {
-		log.Error("Failed to prepareGitCmdWithService: %v", err)
+		ctx.Resp.WriteHeader(http.StatusBadRequest)
-		// FIXME: why it's 401 if the service type doesn't supported?
-		ctx.Resp.WriteHeader(http.StatusUnauthorized)
 		return
 	}
+	// git upload-archive does not have a "--stateless-rpc" option
+	if service == ServiceTypeUploadPack || service == ServiceTypeReceivePack {
+		cmd.AddArguments("--stateless-rpc")
+	}
 
 	ctx.Resp.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", service))
 
@@ -419,10 +430,10 @@ func serviceRPC(ctx *context.Context, h *serviceHandler, service string) {
 
 	// Handle GZIP.
 	if ctx.Req.Header.Get("Content-Encoding") == "gzip" {
+		var err error
 		reqBody, err = gzip.NewReader(reqBody)
 		if err != nil {
-			log.Error("Fail to create gzip reader: %v", err)
+			ctx.Resp.WriteHeader(http.StatusBadRequest)
-			ctx.Resp.WriteHeader(http.StatusInternalServerError)
 			return
 		}
 	}
@@ -435,7 +446,7 @@ func serviceRPC(ctx *context.Context, h *serviceHandler, service string) {
 	}
 
 	var stderr bytes.Buffer
-	if err := gitrepo.RunCmd(ctx, h.getStorageRepo(), cmd.AddArguments("--stateless-rpc", ".").
+	if err := gitrepo.RunCmd(ctx, h.getStorageRepo(), cmd.AddArguments(".").
 		WithEnv(append(os.Environ(), h.environ...)).
 		WithStderr(&stderr).
 		WithStdin(reqBody).
@@ -444,39 +455,27 @@ func serviceRPC(ctx *context.Context, h *serviceHandler, service string) {
 		if !git.IsErrCanceledOrKilled(err) {
 			log.Error("Fail to serve RPC(%s) in %s: %v - %s", service, h.getStorageRepo().RelativePath(), err, stderr.String())
 		}
-		return
 	}
 }
 
 const (
-	ServiceTypeUploadPack  = "upload-pack"
+	ServiceTypeUploadPack    = "upload-pack"
-	ServiceTypeReceivePack = "receive-pack"
+	ServiceTypeReceivePack   = "receive-pack"
+	ServiceTypeUploadArchive = "upload-archive"
 )
 
 // ServiceUploadPack implements Git Smart HTTP protocol
 func ServiceUploadPack(ctx *context.Context) {
-	h := httpBase(ctx)
+	serviceRPC(ctx, ServiceTypeUploadPack)
-	if h != nil {
-		serviceRPC(ctx, h, ServiceTypeUploadPack)
-	}
 }
 
 // ServiceReceivePack implements Git Smart HTTP protocol
 func ServiceReceivePack(ctx *context.Context) {
-	h := httpBase(ctx)
+	serviceRPC(ctx, ServiceTypeReceivePack)
-	if h != nil {
-		serviceRPC(ctx, h, ServiceTypeReceivePack)
-	}
 }
 
-func getServiceType(ctx *context.Context) string {
+func ServiceUploadArchive(ctx *context.Context) {
-	switch ctx.Req.FormValue("service") {
+	serviceRPC(ctx, ServiceTypeUploadArchive)
-	case "git-" + ServiceTypeUploadPack:
-		return ServiceTypeUploadPack
-	case "git-" + ServiceTypeReceivePack:
-		return ServiceTypeReceivePack
-	}
-	return ""
 }
 
 func packetWrite(str string) []byte {
@@ -489,36 +488,45 @@ func packetWrite(str string) []byte {
 
 // GetInfoRefs implements Git dumb HTTP
 func GetInfoRefs(ctx *context.Context) {
-	h := httpBase(ctx)
+	h := httpBase(ctx, ctx.FormString("service")) // git http protocol: "?service=git-<service>"
 	if h == nil {
 		return
 	}
 	setHeaderNoCache(ctx)
-	service := getServiceType(ctx)
+	if h.serviceType == "" {
-	cmd, err := prepareGitCmdWithAllowedService(service)
+		// it's said that some legacy git clients will send requests to "/info/refs" without "service" parameter,
-	if err == nil {
+		// although there should be no such case client in the modern days. TODO: not quite sure why we need this UpdateServerInfo logic
-		if protocol := ctx.Req.Header.Get("Git-Protocol"); protocol != "" && safeGitProtocolHeader.MatchString(protocol) {
-			h.environ = append(h.environ, "GIT_PROTOCOL="+protocol)
-		}
-		h.environ = append(os.Environ(), h.environ...)
-
-		refs, _, err := gitrepo.RunCmdBytes(ctx, h.getStorageRepo(), cmd.AddArguments("--stateless-rpc", "--advertise-refs", ".").
-			WithEnv(h.environ))
-		if err != nil {
-			log.Error(fmt.Sprintf("%v - %s", err, string(refs)))
-		}
-
-		ctx.Resp.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", service))
-		ctx.Resp.WriteHeader(http.StatusOK)
-		_, _ = ctx.Resp.Write(packetWrite("# service=git-" + service + "\n"))
-		_, _ = ctx.Resp.Write([]byte("0000"))
-		_, _ = ctx.Resp.Write(refs)
-	} else {
 		if err := gitrepo.UpdateServerInfo(ctx, h.getStorageRepo()); err != nil {
-			log.Error("Failed to update server info: %v", err)
+			ctx.ServerError("UpdateServerInfo", err)
+			return
 		}
 		h.sendFile(ctx, "text/plain; charset=utf-8", "info/refs")
+		return
 	}
+
+	cmd := prepareGitCmdWithAllowedService(h.serviceType, []string{ServiceTypeUploadPack, ServiceTypeReceivePack})
+	if cmd == nil {
+		ctx.Resp.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if protocol := ctx.Req.Header.Get("Git-Protocol"); protocol != "" && safeGitProtocolHeader.MatchString(protocol) {
+		h.environ = append(h.environ, "GIT_PROTOCOL="+protocol)
+	}
+	h.environ = append(os.Environ(), h.environ...)
+
+	cmd = cmd.AddArguments("--stateless-rpc", "--advertise-refs", ".").WithEnv(h.environ)
+	refs, _, err := gitrepo.RunCmdBytes(ctx, h.getStorageRepo(), cmd)
+	if err != nil {
+		ctx.ServerError("RunGitServiceAdvertiseRefs", err)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", h.serviceType))
+	ctx.Resp.WriteHeader(http.StatusOK)
+	_, _ = ctx.Resp.Write(packetWrite("# service=git-" + h.serviceType + "\n"))
+	_, _ = ctx.Resp.Write([]byte("0000"))
+	_, _ = ctx.Resp.Write(refs)
 }
 
 // GetTextFile implements Git dumb HTTP

tests/integration/git_helper_for_declarative_test.go

@@ -229,3 +229,15 @@ func doGitPull(dstPath string, args ...string) func(*testing.T) {
 		assert.NoError(t, err)
 	}
 }
+
+// doGitRemoteArchive runs a git archive command requesting an archive from remote
+// and verifies that the command did not error out and returned only normal output
+func doGitRemoteArchive(remote string, args ...string) func(*testing.T) {
+	return func(t *testing.T) {
+		stdout, stderr, err := gitcmd.NewCommand("archive").AddOptionValues("--remote", remote).AddArguments(gitcmd.ToTrustedCmdArgs(args)...).
+			RunStdString(t.Context())
+		require.NoError(t, err)
+		assert.Empty(t, stderr)
+		assert.NotEmpty(t, stdout)
+	}
+}

tests/integration/git_smart_http_test.go

@@ -21,6 +21,7 @@ func TestGitSmartHTTP(t *testing.T) {
 	onGiteaRun(t, func(t *testing.T, u *url.URL) {
 		testGitSmartHTTP(t, u)
 		testRenamedRepoRedirect(t)
+		testGitArchiveRemote(t, u)
 	})
 }
 
@@ -96,3 +97,10 @@ func testRenamedRepoRedirect(t *testing.T) {
 	resp = MakeRequest(t, req, http.StatusOK)
 	assert.Contains(t, resp.Body.String(), "65f1bf27bc3bf70f64657658635e66094edbcb4d\trefs/tags/v1.1")
 }
+
+func testGitArchiveRemote(t *testing.T, u *url.URL) {
+	u = u.JoinPath("user27/repo49.git")
+	t.Run("Fetch HEAD archive", doGitRemoteArchive(u.String(), "HEAD"))
+	t.Run("Fetch HEAD archive subpath", doGitRemoteArchive(u.String(), "HEAD", "test"))
+	t.Run("list compression options", doGitRemoteArchive(u.String(), "--list"))
+}