Refactor git command context & pipeline (#36406)

Less and simpler code, fewer bugs
This commit is contained in:
wxiaoguang
2026-01-21 09:35:14 +08:00
committed by GitHub
parent f6db180a80
commit 9ea91e036f
36 changed files with 286 additions and 434 deletions
-3
View File
@@ -737,11 +737,8 @@ LEVEL = Info
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Git Operation timeout in seconds ;; Git Operation timeout in seconds
;[git.timeout] ;[git.timeout]
;DEFAULT = 360
;MIGRATE = 600 ;MIGRATE = 600
;MIRROR = 300 ;MIRROR = 300
;CLONE = 300
;PULL = 300
;GC = 60 ;GC = 60
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+1 -1
View File
@@ -82,7 +82,7 @@ func NewBatchChecker(repo *git.Repository, treeish string, attributes []string)
WithStdout(lw). WithStdout(lw).
RunWithStderr(ctx) RunWithStderr(ctx)
if err != nil && !git.IsErrCanceledOrKilled(err) { if err != nil && !gitcmd.IsErrorCanceledOrKilled(err) {
log.Error("Attribute checker for commit %s exits with error: %v", treeish, err) log.Error("Attribute checker for commit %s exits with error: %v", treeish, err)
} }
checker.cancel() checker.cancel()
+1 -2
View File
@@ -44,8 +44,7 @@ func newCatFileBatch(ctx context.Context, repoPath string, cmdCatFile *gitcmd.Co
cmdCatFile = cmdCatFile. cmdCatFile = cmdCatFile.
WithDir(repoPath). WithDir(repoPath).
WithStdinWriter(&batchStdinWriter). WithStdinWriter(&batchStdinWriter).
WithStdoutReader(&batchStdoutReader). WithStdoutReader(&batchStdoutReader)
WithUseContextTimeout(true)
err := cmdCatFile.StartWithStderr(ctx) err := cmdCatFile.StartWithStderr(ctx)
if err != nil { if err != nil {
+4 -20
View File
@@ -5,10 +5,8 @@ package git
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"io" "io"
"os"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
@@ -284,30 +282,16 @@ func GetAffectedFiles(repo *Repository, branchName, oldCommitID, newCommitID str
} }
oldCommitID = startCommitID oldCommitID = startCommitID
} }
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
log.Error("Unable to create os.Pipe for %s", repo.Path)
return nil, err
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
affectedFiles := make([]string, 0, 32) affectedFiles := make([]string, 0, 32)
// Run `git diff --name-only` to get the names of the changed files // Run `git diff --name-only` to get the names of the changed files
err = gitcmd.NewCommand("diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID). var stdoutReader io.ReadCloser
err := gitcmd.NewCommand("diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID).
WithEnv(env). WithEnv(env).
WithDir(repo.Path). WithDir(repo.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
// Close the writer end of the pipe to begin processing
_ = stdoutWriter.Close()
defer func() {
// Close the reader on return to terminate the git command if necessary
_ = stdoutReader.Close()
}()
// Now scan the output from the command // Now scan the output from the command
scanner := bufio.NewScanner(stdoutReader) scanner := bufio.NewScanner(stdoutReader)
for scanner.Scan() { for scanner.Scan() {
-9
View File
@@ -4,8 +4,6 @@
package git package git
import ( import (
"context"
"errors"
"fmt" "fmt"
"strings" "strings"
@@ -143,10 +141,3 @@ func IsErrMoreThanOne(err error) bool {
func (err *ErrMoreThanOne) Error() string { func (err *ErrMoreThanOne) Error() string {
return fmt.Sprintf("ErrMoreThanOne Error: %v: %s\n%s", err.Err, err.StdErr, err.StdOut) return fmt.Sprintf("ErrMoreThanOne Error: %v: %s\n%s", err.Err, err.StdErr, err.StdOut)
} }
func IsErrCanceledOrKilled(err error) bool {
// When "cancel()" a git command's context, the returned error of "Run()" could be one of them:
// - context.Canceled
// - *exec.ExitError: "signal: killed"
return err != nil && (errors.Is(err, context.Canceled) || err.Error() == "signal: killed")
}
-5
View File
@@ -12,7 +12,6 @@ import (
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings" "strings"
"time"
"code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@@ -140,10 +139,6 @@ func InitSimple() error {
log.Warn("git module has been initialized already, duplicate init may work but it's better to fix it") log.Warn("git module has been initialized already, duplicate init may work but it's better to fix it")
} }
if setting.Git.Timeout.Default > 0 {
gitcmd.SetDefaultCommandExecutionTimeout(time.Duration(setting.Git.Timeout.Default) * time.Second)
}
if err := gitcmd.SetExecutablePath(setting.Git.Path); err != nil { if err := gitcmd.SetExecutablePath(setting.Git.Path); err != nil {
return err return err
} }
+50 -122
View File
@@ -28,13 +28,6 @@ import (
// In most cases, it shouldn't be used. Use AddXxx function instead // In most cases, it shouldn't be used. Use AddXxx function instead
type TrustedCmdArgs []internal.CmdArg type TrustedCmdArgs []internal.CmdArg
// defaultCommandExecutionTimeout default command execution timeout duration
var defaultCommandExecutionTimeout = 360 * time.Second
func SetDefaultCommandExecutionTimeout(timeout time.Duration) {
defaultCommandExecutionTimeout = timeout
}
// DefaultLocale is the default LC_ALL to run git commands in. // DefaultLocale is the default LC_ALL to run git commands in.
const DefaultLocale = "C" const DefaultLocale = "C"
@@ -44,13 +37,14 @@ type Command struct {
prog string prog string
args []string args []string
preErrors []error preErrors []error
cmd *exec.Cmd // for debug purpose only
configArgs []string configArgs []string
opts runOpts opts runOpts
cmd *exec.Cmd
cmdCtx context.Context cmdCtx context.Context
cmdCancel context.CancelFunc cmdCancel process.CancelCauseFunc
cmdFinished context.CancelFunc cmdFinished process.FinishedFunc
cmdStartTime time.Time cmdStartTime time.Time
cmdStdinWriter *io.WriteCloser cmdStdinWriter *io.WriteCloser
@@ -209,11 +203,9 @@ func ToTrustedCmdArgs(args []string) TrustedCmdArgs {
return ret return ret
} }
// runOpts represents parameters to run the command. If UseContextTimeout is specified, then Timeout is ignored.
type runOpts struct { type runOpts struct {
Env []string Env []string
Timeout time.Duration Timeout time.Duration
UseContextTimeout bool
// Dir is the working dir for the git command, however: // Dir is the working dir for the git command, however:
// FIXME: this could be incorrect in many cases, for example: // FIXME: this could be incorrect in many cases, for example:
@@ -236,7 +228,7 @@ type runOpts struct {
// Use new functions like WithStdinWriter to avoid such problems. // Use new functions like WithStdinWriter to avoid such problems.
Stdin io.Reader Stdin io.Reader
PipelineFunc func(context.Context, context.CancelFunc) error PipelineFunc func(Context) error
} }
func commonBaseEnvs() []string { func commonBaseEnvs() []string {
@@ -321,16 +313,11 @@ func (c *Command) WithStdin(stdin io.Reader) *Command {
return c return c
} }
func (c *Command) WithPipelineFunc(f func(context.Context, context.CancelFunc) error) *Command { func (c *Command) WithPipelineFunc(f func(Context) error) *Command {
c.opts.PipelineFunc = f c.opts.PipelineFunc = f
return c return c
} }
func (c *Command) WithUseContextTimeout(useContextTimeout bool) *Command {
c.opts.UseContextTimeout = useContextTimeout
return c
}
// WithParentCallerInfo can be used to set the caller info (usually function name) of the parent function of the caller. // WithParentCallerInfo can be used to set the caller info (usually function name) of the parent function of the caller.
// For most cases, "Run" family functions can get its caller info automatically // For most cases, "Run" family functions can get its caller info automatically
// But if you need to call "Run" family functions in a wrapper function: "FeatureFunc -> GeneralWrapperFunc -> RunXxx", // But if you need to call "Run" family functions in a wrapper function: "FeatureFunc -> GeneralWrapperFunc -> RunXxx",
@@ -363,9 +350,7 @@ func (c *Command) Start(ctx context.Context) (retErr error) {
defer func() { defer func() {
if retErr != nil { if retErr != nil {
// release the pipes to avoid resource leak // release the pipes to avoid resource leak
safeClosePtrCloser(c.cmdStdoutReader) c.closeStdioPipes()
safeClosePtrCloser(c.cmdStderrReader)
safeClosePtrCloser(c.cmdStdinWriter)
// if error occurs, we must also finish the task, otherwise, cmdFinished will be called in "Wait" function // if error occurs, we must also finish the task, otherwise, cmdFinished will be called in "Wait" function
if c.cmdFinished != nil { if c.cmdFinished != nil {
c.cmdFinished() c.cmdFinished()
@@ -380,12 +365,6 @@ func (c *Command) Start(ctx context.Context) (retErr error) {
return err return err
} }
// We must not change the provided options
timeout := c.opts.Timeout
if timeout <= 0 {
timeout = defaultCommandExecutionTimeout
}
cmdLogString := c.LogString() cmdLogString := c.LogString()
if c.callerInfo == "" { if c.callerInfo == "" {
c.WithParentCallerInfo() c.WithParentCallerInfo()
@@ -399,83 +378,85 @@ func (c *Command) Start(ctx context.Context) (retErr error) {
span.SetAttributeString(gtprof.TraceAttrFuncCaller, c.callerInfo) span.SetAttributeString(gtprof.TraceAttrFuncCaller, c.callerInfo)
span.SetAttributeString(gtprof.TraceAttrGitCommand, cmdLogString) span.SetAttributeString(gtprof.TraceAttrGitCommand, cmdLogString)
if c.opts.UseContextTimeout { if c.opts.Timeout <= 0 {
c.cmdCtx, c.cmdCancel, c.cmdFinished = process.GetManager().AddContext(ctx, desc) c.cmdCtx, c.cmdCancel, c.cmdFinished = process.GetManager().AddContext(ctx, desc)
} else { } else {
c.cmdCtx, c.cmdCancel, c.cmdFinished = process.GetManager().AddContextTimeout(ctx, timeout, desc) c.cmdCtx, c.cmdCancel, c.cmdFinished = process.GetManager().AddContextTimeout(ctx, c.opts.Timeout, desc)
} }
c.cmdStartTime = time.Now() c.cmdStartTime = time.Now()
cmd := exec.CommandContext(ctx, c.prog, append(c.configArgs, c.args...)...) c.cmd = exec.CommandContext(ctx, c.prog, append(c.configArgs, c.args...)...)
c.cmd = cmd // for debug purpose only
if c.opts.Env == nil { if c.opts.Env == nil {
cmd.Env = os.Environ() c.cmd.Env = os.Environ()
} else { } else {
cmd.Env = c.opts.Env c.cmd.Env = c.opts.Env
} }
process.SetSysProcAttribute(cmd) process.SetSysProcAttribute(c.cmd)
cmd.Env = append(cmd.Env, CommonGitCmdEnvs()...) c.cmd.Env = append(c.cmd.Env, CommonGitCmdEnvs()...)
cmd.Dir = c.opts.Dir c.cmd.Dir = c.opts.Dir
cmd.Stdout = c.opts.Stdout c.cmd.Stdout = c.opts.Stdout
cmd.Stdin = c.opts.Stdin c.cmd.Stdin = c.opts.Stdin
if _, err := safeAssignPipe(c.cmdStdinWriter, cmd.StdinPipe); err != nil { if _, err := safeAssignPipe(c.cmdStdinWriter, c.cmd.StdinPipe); err != nil {
return err return err
} }
if _, err := safeAssignPipe(c.cmdStdoutReader, cmd.StdoutPipe); err != nil { if _, err := safeAssignPipe(c.cmdStdoutReader, c.cmd.StdoutPipe); err != nil {
return err return err
} }
if _, err := safeAssignPipe(c.cmdStderrReader, cmd.StderrPipe); err != nil { if _, err := safeAssignPipe(c.cmdStderrReader, c.cmd.StderrPipe); err != nil {
return err return err
} }
if c.cmdManagedStderr != nil { if c.cmdManagedStderr != nil {
if cmd.Stderr != nil { if c.cmd.Stderr != nil {
panic("CombineStderr needs managed (but not caller-provided) stderr pipe") panic("CombineStderr needs managed (but not caller-provided) stderr pipe")
} }
cmd.Stderr = c.cmdManagedStderr c.cmd.Stderr = c.cmdManagedStderr
} }
return cmd.Start() return c.cmd.Start()
}
func (c *Command) closeStdioPipes() {
safeClosePtrCloser(c.cmdStdoutReader)
safeClosePtrCloser(c.cmdStderrReader)
safeClosePtrCloser(c.cmdStdinWriter)
} }
func (c *Command) Wait() error { func (c *Command) Wait() error {
defer func() { defer func() {
safeClosePtrCloser(c.cmdStdoutReader) c.closeStdioPipes()
safeClosePtrCloser(c.cmdStderrReader)
safeClosePtrCloser(c.cmdStdinWriter)
c.cmdFinished() c.cmdFinished()
}() }()
cmd, ctx, cancel := c.cmd, c.cmdCtx, c.cmdCancel
if c.opts.PipelineFunc != nil { if c.opts.PipelineFunc != nil {
err := c.opts.PipelineFunc(ctx, cancel) errCallback := c.opts.PipelineFunc(&cmdContext{Context: c.cmdCtx, cmd: c})
if err != nil { // after the pipeline function returns, we can safely cancel the command context and close the stdio pipes
cancel() c.cmdCancel(errCallback)
errWait := cmd.Wait() c.closeStdioPipes()
return errors.Join(err, errWait) errWait := c.cmd.Wait()
errCause := context.Cause(c.cmdCtx)
// the pipeline function should be able to know whether it succeeds or fails
if errCallback == nil && (errCause == nil || errors.Is(errCause, context.Canceled)) {
return nil
} }
return errors.Join(errCallback, errCause, errWait)
} }
errWait := cmd.Wait() // there might be other goroutines using the context or pipes, so we just wait for the command to finish
errWait := c.cmd.Wait()
elapsed := time.Since(c.cmdStartTime) elapsed := time.Since(c.cmdStartTime)
if elapsed > time.Second { if elapsed > time.Second {
log.Debug("slow git.Command.Run: %s (%s)", c, elapsed) log.Debug("slow git.Command.Run: %s (%s)", c, elapsed) // TODO: no need to log this for long-running commands
} }
// Here the logic is different from "PipelineFunc" case,
// because PipelineFunc can return error if it fails, it knows whether it succeeds or fails.
// But in normal case, the caller just runs the git command, the command's exit code is the source of truth.
// If the caller need to know whether the command error is caused by cancellation, it should check the "err" by itself.
errCause := context.Cause(c.cmdCtx) errCause := context.Cause(c.cmdCtx)
if errors.Is(errCause, context.Canceled) { return errors.Join(errCause, errWait)
// if the ctx is canceled without other error, it must be caused by normal cancellation
return errCause
}
if errWait != nil {
// no matter whether there is other cause error, if "Wait" also has error,
// it's likely the error is caused by Wait error (from git command)
return errWait
}
return errCause
} }
func (c *Command) StartWithStderr(ctx context.Context) RunStdError { func (c *Command) StartWithStderr(ctx context.Context) RunStdError {
@@ -513,59 +494,6 @@ func (c *Command) Run(ctx context.Context) (err error) {
return c.Wait() return c.Wait()
} }
type RunStdError interface {
error
Unwrap() error
Stderr() string
}
type runStdError struct {
err error // usually the low-level error like `*exec.ExitError`
stderr string // git command's stderr output
errMsg string // the cached error message for Error() method
}
func (r *runStdError) Error() string {
// FIXME: GIT-CMD-STDERR: it is a bad design, the stderr should not be put in the error message
// But a lot of code only checks `strings.Contains(err.Error(), "git error")`
if r.errMsg == "" {
r.errMsg = fmt.Sprintf("%s - %s", r.err.Error(), strings.TrimSpace(r.stderr))
}
return r.errMsg
}
func (r *runStdError) Unwrap() error {
return r.err
}
func (r *runStdError) Stderr() string {
return r.stderr
}
func ErrorAsStderr(err error) (string, bool) {
var runErr RunStdError
if errors.As(err, &runErr) {
return runErr.Stderr(), true
}
return "", false
}
func StderrHasPrefix(err error, prefix string) bool {
stderr, ok := ErrorAsStderr(err)
if !ok {
return false
}
return strings.HasPrefix(stderr, prefix)
}
func IsErrorExitCode(err error, code int) bool {
var exitError *exec.ExitError
if errors.As(err, &exitError) {
return exitError.ExitCode() == code
}
return false
}
// RunStdString runs the command and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). // RunStdString runs the command and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr).
func (c *Command) RunStdString(ctx context.Context) (stdout, stderr string, runErr RunStdError) { func (c *Command) RunStdString(ctx context.Context) (stdout, stderr string, runErr RunStdError) {
stdoutBytes, stderrBytes, runErr := c.WithParentCallerInfo().runStdBytes(ctx) stdoutBytes, stderrBytes, runErr := c.WithParentCallerInfo().runStdBytes(ctx)
-38
View File
@@ -1,38 +0,0 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build race
package gitcmd
import (
"context"
"testing"
"time"
)
func TestRunWithContextNoTimeout(t *testing.T) {
maxLoops := 10
// 'git --version' does not block so it must be finished before the timeout triggered.
for i := 0; i < maxLoops; i++ {
cmd := NewCommand("--version")
if err := cmd.Run(t.Context()); err != nil {
t.Fatal(err)
}
}
}
func TestRunWithContextTimeout(t *testing.T) {
maxLoops := 10
// 'git hash-object --stdin' blocks on stdin so we can have the timeout triggered.
for i := 0; i < maxLoops; i++ {
cmd := NewCommand("hash-object", "--stdin")
if err := cmd.WithTimeout(1 * time.Millisecond).Run(t.Context()); err != nil {
if err != context.DeadlineExceeded {
t.Fatalf("Testing %d/%d: %v", i, maxLoops, err)
}
}
}
}
+14
View File
@@ -4,9 +4,11 @@
package gitcmd package gitcmd
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"testing" "testing"
"time"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/tempdir"
@@ -111,3 +113,15 @@ func TestRunStdError(t *testing.T) {
require.ErrorAs(t, fmt.Errorf("wrapped %w", err), &asErr) require.ErrorAs(t, fmt.Errorf("wrapped %w", err), &asErr)
} }
func TestRunWithContextTimeout(t *testing.T) {
t.Run("NoTimeout", func(t *testing.T) {
// 'git --version' does not block so it must be finished before the timeout triggered.
err := NewCommand("--version").Run(t.Context())
require.NoError(t, err)
})
t.Run("WithTimeout", func(t *testing.T) {
err := NewCommand("hash-object", "--stdin").WithTimeout(1 * time.Millisecond).Run(t.Context())
require.ErrorIs(t, err, context.DeadlineExceeded)
})
}
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitcmd
import (
"context"
)
type Context interface {
context.Context
// CancelWithCause is a helper function to cancel the context with a specific error cause
// And it returns the same error for convenience, to break the PipelineFunc easily
CancelWithCause(err error) error
// In the future, this interface will be extended to support stdio pipe readers/writers
}
type cmdContext struct {
context.Context
cmd *Command
}
func (c *cmdContext) CancelWithCause(err error) error {
c.cmd.cmdCancel(err)
return err
}
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitcmd
import (
"context"
"errors"
"fmt"
"os/exec"
"strings"
)
type RunStdError interface {
error
Unwrap() error
Stderr() string
}
type runStdError struct {
err error // usually the low-level error like `*exec.ExitError`
stderr string // git command's stderr output
errMsg string // the cached error message for Error() method
}
func (r *runStdError) Error() string {
// FIXME: GIT-CMD-STDERR: it is a bad design, the stderr should not be put in the error message
// But a lot of code only checks `strings.Contains(err.Error(), "git error")`
if r.errMsg == "" {
r.errMsg = fmt.Sprintf("%s - %s", r.err.Error(), strings.TrimSpace(r.stderr))
}
return r.errMsg
}
func (r *runStdError) Unwrap() error {
return r.err
}
func (r *runStdError) Stderr() string {
return r.stderr
}
func ErrorAsStderr(err error) (string, bool) {
var runErr RunStdError
if errors.As(err, &runErr) {
return runErr.Stderr(), true
}
return "", false
}
func StderrHasPrefix(err error, prefix string) bool {
stderr, ok := ErrorAsStderr(err)
if !ok {
return false
}
return strings.HasPrefix(stderr, prefix)
}
func IsErrorExitCode(err error, code int) bool {
var exitError *exec.ExitError
if errors.As(err, &exitError) {
return exitError.ExitCode() == code
}
return false
}
func IsErrorSignalKilled(err error) bool {
var exitError *exec.ExitError
return errors.As(err, &exitError) && exitError.String() == "signal: killed"
}
func IsErrorCanceledOrKilled(err error) bool {
// When "cancel()" a git command's context, the returned error of "Run()" could be one of them:
// - context.Canceled
// - *exec.ExitError: "signal: killed"
// TODO: in the future, we need to use unified error type from gitcmd.Run to check whether it is manually canceled
return errors.Is(err, context.Canceled) || IsErrorSignalKilled(err)
}
+2 -5
View File
@@ -77,9 +77,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
var stdoutReader io.ReadCloser var stdoutReader io.ReadCloser
err := cmd.WithDir(repo.Path). err := cmd.WithDir(repo.Path).
WithStdoutReader(&stdoutReader). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
defer stdoutReader.Close()
isInBlock := false isInBlock := false
rd := bufio.NewReaderSize(stdoutReader, util.IfZero(opts.MaxLineLength, 16*1024)) rd := bufio.NewReaderSize(stdoutReader, util.IfZero(opts.MaxLineLength, 16*1024))
var res *GrepResult var res *GrepResult
@@ -105,8 +103,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
} }
if line == "" { if line == "" {
if len(results) >= opts.MaxResultLimit { if len(results) >= opts.MaxResultLimit {
cancel() return ctx.CancelWithCause(nil)
break
} }
isInBlock = false isInBlock = false
continue continue
+3 -3
View File
@@ -74,9 +74,9 @@ func (err *ErrInvalidCloneAddr) Unwrap() error {
func IsRemoteNotExistError(err error) bool { func IsRemoteNotExistError(err error) bool {
// see: https://github.com/go-gitea/gitea/issues/32889#issuecomment-2571848216 // see: https://github.com/go-gitea/gitea/issues/32889#issuecomment-2571848216
// Should not add space in the end, sometimes git will add a `:` // Should not add space in the end, sometimes git will add a `:`
prefix1 := "exit status 128 - fatal: No such remote" // git < 2.30 prefix1 := "fatal: No such remote" // git < 2.30, exit status 128
prefix2 := "exit status 2 - error: No such remote" // git >= 2.30 prefix2 := "error: No such remote" // git >= 2.30. exit status 2
return strings.HasPrefix(err.Error(), prefix1) || strings.HasPrefix(err.Error(), prefix2) return gitcmd.StderrHasPrefix(err, prefix1) || gitcmd.StderrHasPrefix(err, prefix2)
} }
// ParseRemoteAddr checks if given remote address is valid, // ParseRemoteAddr checks if given remote address is valid,
+4 -14
View File
@@ -5,9 +5,8 @@ package git
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"os" "io"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -55,15 +54,6 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string)
} }
stats.CommitCountInAllBranches = c stats.CommitCountInAllBranches = c
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, err
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
gitCmd := gitcmd.NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso"). gitCmd := gitcmd.NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso").
AddOptionFormat("--since=%s", since) AddOptionFormat("--since=%s", since)
if len(branch) == 0 { if len(branch) == 0 {
@@ -72,11 +62,11 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string)
gitCmd.AddArguments("--first-parent").AddDynamicArguments(branch) gitCmd.AddArguments("--first-parent").AddDynamicArguments(branch)
} }
var stdoutReader io.ReadCloser
err = gitCmd. err = gitCmd.
WithDir(repo.Path). WithDir(repo.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
scanner := bufio.NewScanner(stdoutReader) scanner := bufio.NewScanner(stdoutReader)
scanner.Split(bufio.ScanLines) scanner.Split(bufio.ScanLines)
stats.CommitCount = 0 stats.CommitCount = 0
+5 -13
View File
@@ -7,7 +7,7 @@ import (
"bufio" "bufio"
"context" "context"
"fmt" "fmt"
"os" "io"
"code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@@ -21,23 +21,15 @@ type TemplateSubmoduleCommit struct {
// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository // GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository
// This function is only for generating new repos based on existing template, the template couldn't be too large. // This function is only for generating new repos based on existing template, the template couldn't be too large.
func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) { func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) {
stdoutReader, stdoutWriter, err := os.Pipe() var stdoutReader io.ReadCloser
if err != nil { err := gitcmd.NewCommand("ls-tree", "-r", "--", "HEAD").
return nil, err
}
err = gitcmd.NewCommand("ls-tree", "-r", "--", "HEAD").
WithDir(repoPath). WithDir(repoPath).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
defer stdoutReader.Close()
scanner := bufio.NewScanner(stdoutReader) scanner := bufio.NewScanner(stdoutReader)
for scanner.Scan() { for scanner.Scan() {
entry, err := parseLsTreeLine(scanner.Bytes()) entry, err := parseLsTreeLine(scanner.Bytes())
if err != nil { if err != nil {
cancel()
return err return err
} }
if entry.EntryMode == EntryModeCommit { if entry.EntryMode == EntryModeCommit {
+1 -1
View File
@@ -174,7 +174,7 @@ func CreateBlameReader(ctx context.Context, objectFormat git.ObjectFormat, repo
} }
go func() { go func() {
// TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close"
err := RunCmdWithStderr(ctx, repo, cmd.WithUseContextTimeout(true).WithStdout(stdout)) err := RunCmdWithStderr(ctx, repo, cmd.WithStdout(stdout))
done <- err done <- err
_ = stdout.Close() _ = stdout.Close()
}() }()
+3 -3
View File
@@ -35,10 +35,10 @@ func init() {
} }
} }
func newProcessTypedContext(parent context.Context, desc string) (ctx context.Context, cancel context.CancelFunc) { func newProcessTypedContext(parent context.Context, desc string) (context.Context, context.CancelFunc) {
// the "process manager" also calls "log.Trace()" to output logs, so if we want to create new contexts by the manager, we need to disable the trace temporarily // the "process manager" also calls "log.Trace()" to output logs, so if we want to create new contexts by the manager, we need to disable the trace temporarily
process.TraceLogDisable(true) process.TraceLogDisable(true)
defer process.TraceLogDisable(false) defer process.TraceLogDisable(false)
ctx, _, cancel = process.GetManager().AddTypedContext(parent, desc, process.SystemProcessType, false) ctx, _, finished := process.GetManager().AddTypedContext(parent, desc, process.SystemProcessType, false)
return ctx, cancel return ctx, context.CancelFunc(finished)
} }
+1 -1
View File
@@ -20,7 +20,7 @@ var manager *Manager
// Manager is the nosql connection manager // Manager is the nosql connection manager
type Manager struct { type Manager struct {
ctx context.Context ctx context.Context
finished context.CancelFunc finished process.FinishedFunc
mutex sync.Mutex mutex sync.Mutex
RedisConnections map[string]*redisClientHolder RedisConnections map[string]*redisClientHolder
+29 -22
View File
@@ -13,6 +13,7 @@ import (
"time" "time"
"code.gitea.io/gitea/modules/gtprof" "code.gitea.io/gitea/modules/gtprof"
"code.gitea.io/gitea/modules/util"
) )
// TODO: This packages still uses a singleton for the Manager. // TODO: This packages still uses a singleton for the Manager.
@@ -27,12 +28,14 @@ var (
DefaultContext = context.Background() DefaultContext = context.Background()
) )
// IDType is a pid type type (
type IDType string // IDType is a pid type
IDType string
// FinishedFunc is a function that marks that the process is finished and can be removed from the process table CancelCauseFunc func(cause ...error)
// - it is simply an alias for context.CancelFunc and is only for documentary purposes // FinishedFunc is a function that marks that the process is finished and can be removed from the process table
type FinishedFunc = context.CancelFunc FinishedFunc func()
)
var ( var (
traceDisabled atomic.Int64 traceDisabled atomic.Int64
@@ -84,6 +87,10 @@ func GetManager() *Manager {
return manager return manager
} }
func cancelCauseFunc(cancelCause context.CancelCauseFunc) CancelCauseFunc {
return func(cause ...error) { cancelCause(util.OptionalArg(cause)) }
}
// AddContext creates a new context and adds it as a process. Once the process is finished, finished must be called // AddContext creates a new context and adds it as a process. Once the process is finished, finished must be called
// to remove the process from the process table. It should not be called until the process is finished but must always be called. // to remove the process from the process table. It should not be called until the process is finished but must always be called.
// //
@@ -92,11 +99,10 @@ func GetManager() *Manager {
// //
// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
// process table. // process table.
func (pm *Manager) AddContext(parent context.Context, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { func (pm *Manager) AddContext(parent context.Context, description string) (context.Context, CancelCauseFunc, FinishedFunc) {
ctx, cancel = context.WithCancel(parent) ctx, ctxCancel := context.WithCancelCause(parent)
cancel := cancelCauseFunc(ctxCancel)
ctx, _, finished = pm.Add(ctx, description, cancel, NormalProcessType, true) ctx, _, finished := pm.Add(ctx, description, cancel, NormalProcessType, true)
return ctx, cancel, finished return ctx, cancel, finished
} }
@@ -108,11 +114,10 @@ func (pm *Manager) AddContext(parent context.Context, description string) (ctx c
// //
// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
// process table. // process table.
func (pm *Manager) AddTypedContext(parent context.Context, description, processType string, currentlyRunning bool) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { func (pm *Manager) AddTypedContext(parent context.Context, description, processType string, currentlyRunning bool) (context.Context, CancelCauseFunc, FinishedFunc) {
ctx, cancel = context.WithCancel(parent) ctx, ctxCancel := context.WithCancelCause(parent)
cancel := cancelCauseFunc(ctxCancel)
ctx, _, finished = pm.Add(ctx, description, cancel, processType, currentlyRunning) ctx, _, finished := pm.Add(ctx, description, cancel, processType, currentlyRunning)
return ctx, cancel, finished return ctx, cancel, finished
} }
@@ -124,21 +129,23 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT
// //
// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
// process table. // process table.
func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (context.Context, CancelCauseFunc, FinishedFunc) {
if timeout <= 0 { if timeout <= 0 {
// it's meaningless to use timeout <= 0, and it must be a bug! so we must panic here to tell developers to make the timeout correct // it's meaningless to use timeout <= 0, and it must be a bug! so we must panic here to tell developers to make the timeout correct
panic("the timeout must be greater than zero, otherwise the context will be cancelled immediately") panic("the timeout must be greater than zero, otherwise the context will be cancelled immediately")
} }
ctx, ctxCancelTimeout := context.WithTimeout(parent, timeout)
ctx, cancel = context.WithTimeout(parent, timeout) ctx, ctxCancelCause := context.WithCancelCause(ctx)
cancel := func(cause ...error) {
ctx, _, finished = pm.Add(ctx, description, cancel, NormalProcessType, true) ctxCancelCause(util.OptionalArg(cause))
ctxCancelTimeout()
}
ctx, _, finished := pm.Add(ctx, description, cancel, NormalProcessType, true)
return ctx, cancel, finished return ctx, cancel, finished
} }
// Add create a new process // Add create a new process
func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc, processType string, currentlyRunning bool) (context.Context, IDType, FinishedFunc) { func (pm *Manager) Add(ctx context.Context, description string, cancel CancelCauseFunc, processType string, currentlyRunning bool) (context.Context, IDType, FinishedFunc) {
parentPID := GetParentPID(ctx) parentPID := GetParentPID(ctx)
pm.mutex.Lock() pm.mutex.Lock()
+1 -2
View File
@@ -4,7 +4,6 @@
package process package process
import ( import (
"context"
"time" "time"
) )
@@ -21,7 +20,7 @@ type process struct {
ParentPID IDType ParentPID IDType
Description string Description string
Start time.Time Start time.Time
Cancel context.CancelFunc Cancel CancelCauseFunc
Type string Type string
} }
+1 -1
View File
@@ -21,7 +21,7 @@ import (
// It can use different underlying (base) queue types // It can use different underlying (base) queue types
type WorkerPoolQueue[T any] struct { type WorkerPoolQueue[T any] struct {
ctxRun context.Context ctxRun context.Context
ctxRunCancel context.CancelFunc ctxRunCancel process.FinishedFunc
shutdownDone chan struct{} shutdownDone chan struct{}
shutdownTimeout atomic.Int64 // in case some buggy handlers (workers) would hang forever, "shutdown" should finish in predictable time shutdownTimeout atomic.Int64 // in case some buggy handlers (workers) would hang forever, "shutdown" should finish in predictable time
-9
View File
@@ -33,11 +33,8 @@ var Git = struct {
DisablePartialClone bool DisablePartialClone bool
DiffRenameSimilarityThreshold string DiffRenameSimilarityThreshold string
Timeout struct { Timeout struct {
Default int
Migrate int Migrate int
Mirror int Mirror int
Clone int
Pull int
GC int `ini:"GC"` GC int `ini:"GC"`
} `ini:"git.timeout"` } `ini:"git.timeout"`
}{ }{
@@ -56,18 +53,12 @@ var Git = struct {
DisablePartialClone: false, DisablePartialClone: false,
DiffRenameSimilarityThreshold: "50%", DiffRenameSimilarityThreshold: "50%",
Timeout: struct { Timeout: struct {
Default int
Migrate int Migrate int
Mirror int Mirror int
Clone int
Pull int
GC int `ini:"GC"` GC int `ini:"GC"`
}{ }{
Default: 360,
Migrate: 600, Migrate: 600,
Mirror: 300, Mirror: 300,
Clone: 300,
Pull: 300,
GC: 60, GC: 60,
}, },
} }
-2
View File
@@ -3284,8 +3284,6 @@
"admin.config.git_gc_args": "GC Arguments", "admin.config.git_gc_args": "GC Arguments",
"admin.config.git_migrate_timeout": "Migration Timeout", "admin.config.git_migrate_timeout": "Migration Timeout",
"admin.config.git_mirror_timeout": "Mirror Update Timeout", "admin.config.git_mirror_timeout": "Mirror Update Timeout",
"admin.config.git_clone_timeout": "Clone Operation Timeout",
"admin.config.git_pull_timeout": "Pull Operation Timeout",
"admin.config.git_gc_timeout": "GC Operation Timeout", "admin.config.git_gc_timeout": "GC Operation Timeout",
"admin.config.log_config": "Log Configuration", "admin.config.log_config": "Log Configuration",
"admin.config.logger_name_fmt": "Logger: %s", "admin.config.logger_name_fmt": "Logger: %s",
+9 -40
View File
@@ -5,9 +5,7 @@ package private
import ( import (
"bufio" "bufio"
"context"
"io" "io"
"os"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/git/gitcmd"
@@ -18,16 +16,6 @@ import (
// This file contains commit verification functions for refs passed across in hooks // This file contains commit verification functions for refs passed across in hooks
func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error { func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error {
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
log.Error("Unable to create os.Pipe for %s", repo.Path)
return err
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
var command *gitcmd.Command var command *gitcmd.Command
objectFormat, _ := repo.GetObjectFormat() objectFormat, _ := repo.GetObjectFormat()
if oldCommitID == objectFormat.EmptyObjectID().String() { if oldCommitID == objectFormat.EmptyObjectID().String() {
@@ -39,18 +27,13 @@ func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []
command = gitcmd.NewCommand("rev-list").AddDynamicArguments(oldCommitID + "..." + newCommitID) command = gitcmd.NewCommand("rev-list").AddDynamicArguments(oldCommitID + "..." + newCommitID)
} }
// This is safe as force pushes are already forbidden // This is safe as force pushes are already forbidden
err = command.WithEnv(env). var stdoutReader io.ReadCloser
err := command.WithEnv(env).
WithDir(repo.Path). WithDir(repo.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
err := readAndVerifyCommitsFromShaReader(stdoutReader, repo, env) err := readAndVerifyCommitsFromShaReader(stdoutReader, repo, env)
if err != nil { return ctx.CancelWithCause(err)
log.Error("readAndVerifyCommitsFromShaReader failed: %v", err)
cancel()
}
_ = stdoutReader.Close()
return err
}). }).
Run(repo.Ctx) Run(repo.Ctx)
if err != nil && !isErrUnverifiedCommit(err) { if err != nil && !isErrUnverifiedCommit(err) {
@@ -72,34 +55,20 @@ func readAndVerifyCommitsFromShaReader(input io.ReadCloser, repo *git.Repository
} }
func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error { func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error {
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
log.Error("Unable to create pipe for %s: %v", repo.Path, err)
return err
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
commitID := git.MustIDFromString(sha) commitID := git.MustIDFromString(sha)
var stdoutReader io.ReadCloser
return gitcmd.NewCommand("cat-file", "commit").AddDynamicArguments(sha). return gitcmd.NewCommand("cat-file", "commit").AddDynamicArguments(sha).
WithEnv(env). WithEnv(env).
WithDir(repo.Path). WithDir(repo.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
commit, err := git.CommitFromReader(repo, commitID, stdoutReader) commit, err := git.CommitFromReader(repo, commitID, stdoutReader)
if err != nil { if err != nil {
return err return err
} }
verification := asymkey_service.ParseCommitWithSignature(ctx, commit) verification := asymkey_service.ParseCommitWithSignature(ctx, commit)
if !verification.Verified { if !verification.Verified {
cancel() return ctx.CancelWithCause(&errUnverifiedCommit{commit.ID.String()})
return &errUnverifiedCommit{
commit.ID.String(),
}
} }
return nil return nil
}). }).
+3 -3
View File
@@ -447,9 +447,9 @@ func serviceRPC(ctx *context.Context, service string) {
if err := gitrepo.RunCmdWithStderr(ctx, h.getStorageRepo(), cmd.AddArguments("."). if err := gitrepo.RunCmdWithStderr(ctx, h.getStorageRepo(), cmd.AddArguments(".").
WithEnv(append(os.Environ(), h.environ...)). WithEnv(append(os.Environ(), h.environ...)).
WithStdin(reqBody). WithStdin(reqBody).
WithStdout(ctx.Resp). WithStdout(ctx.Resp),
WithUseContextTimeout(true)); err != nil { ); err != nil {
if !git.IsErrCanceledOrKilled(err) { if !gitcmd.IsErrorCanceledOrKilled(err) {
log.Error("Fail to serve RPC(%s) in %s: %v", service, h.getStorageRepo().RelativePath(), err) log.Error("Fail to serve RPC(%s) in %s: %v", service, h.getStorageRepo().RelativePath(), err)
} }
} }
+1 -1
View File
@@ -54,7 +54,7 @@ func registerRepoHealthCheck() {
RunAtStart: false, RunAtStart: false,
Schedule: "@midnight", Schedule: "@midnight",
}, },
Timeout: time.Duration(setting.Git.Timeout.Default) * time.Second, Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second,
Args: []string{}, Args: []string{},
}, func(ctx context.Context, _ *user_model.User, config Config) error { }, func(ctx context.Context, _ *user_model.User, config Config) error {
rhcConfig := config.(*RepoHealthCheckConfig) rhcConfig := config.(*RepoHealthCheckConfig)
+2 -3
View File
@@ -16,7 +16,6 @@ import (
"path" "path"
"sort" "sort"
"strings" "strings"
"time"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git" git_model "code.gitea.io/gitea/models/git"
@@ -1271,10 +1270,10 @@ func getDiffBasic(ctx context.Context, gitRepo *git.Repository, opts *DiffOption
}() }()
go func() { go func() {
if err := cmdDiff.WithTimeout(time.Duration(setting.Git.Timeout.Default) * time.Second). if err := cmdDiff.
WithDir(repoPath). WithDir(repoPath).
WithStdout(writer). WithStdout(writer).
RunWithStderr(cmdCtx); err != nil && !git.IsErrCanceledOrKilled(err) { RunWithStderr(cmdCtx); err != nil && !gitcmd.IsErrorCanceledOrKilled(err) {
log.Error("error during GetDiff(git diff dir: %s): %v", repoPath, err) log.Error("error during GetDiff(git diff dir: %s): %v", repoPath, err)
} }
+9 -23
View File
@@ -206,16 +206,6 @@ func prepareTemporaryRepoForMerge(ctx *mergeContext) error {
// getDiffTree returns a string containing all the files that were changed between headBranch and baseBranch // getDiffTree returns a string containing all the files that were changed between headBranch and baseBranch
// the filenames are escaped so as to fit the format required for .git/info/sparse-checkout // the filenames are escaped so as to fit the format required for .git/info/sparse-checkout
func getDiffTree(ctx context.Context, repoPath, baseBranch, headBranch string, out io.Writer) error { func getDiffTree(ctx context.Context, repoPath, baseBranch, headBranch string, out io.Writer) error {
diffOutReader, diffOutWriter, err := os.Pipe()
if err != nil {
log.Error("Unable to create os.Pipe for %s", repoPath)
return err
}
defer func() {
_ = diffOutReader.Close()
_ = diffOutWriter.Close()
}()
scanNullTerminatedStrings := func(data []byte, atEOF bool) (advance int, token []byte, err error) { scanNullTerminatedStrings := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 { if atEOF && len(data) == 0 {
return 0, nil, nil return 0, nil, nil
@@ -229,27 +219,23 @@ func getDiffTree(ctx context.Context, repoPath, baseBranch, headBranch string, o
return 0, nil, nil return 0, nil, nil
} }
err = gitcmd.NewCommand("diff-tree", "--no-commit-id", "--name-only", "-r", "-r", "-z", "--root"). var diffOutReader io.ReadCloser
err := gitcmd.NewCommand("diff-tree", "--no-commit-id", "--name-only", "-r", "-r", "-z", "--root").
AddDynamicArguments(baseBranch, headBranch). AddDynamicArguments(baseBranch, headBranch).
WithDir(repoPath). WithDir(repoPath).
WithStdout(diffOutWriter). WithStdoutReader(&diffOutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
// Close the writer end of the pipe to begin processing
_ = diffOutWriter.Close()
defer func() {
// Close the reader on return to terminate the git command if necessary
_ = diffOutReader.Close()
}()
// Now scan the output from the command // Now scan the output from the command
scanner := bufio.NewScanner(diffOutReader) scanner := bufio.NewScanner(diffOutReader)
scanner.Split(scanNullTerminatedStrings) scanner.Split(scanNullTerminatedStrings)
for scanner.Scan() { for scanner.Scan() {
filepath := scanner.Text() treePath := scanner.Text()
// escape '*', '?', '[', spaces and '!' prefix // escape '*', '?', '[', spaces and '!' prefix
filepath = escapedSymbols.ReplaceAllString(filepath, `\$1`) treePath = escapedSymbols.ReplaceAllString(treePath, `\$1`)
// no necessary to escape the first '#' symbol because the first symbol is '/' // no necessary to escape the first '#' symbol because the first symbol is '/'
fmt.Fprintf(out, "/%s\n", filepath) if _, err := fmt.Fprintf(out, "/%s\n", treePath); err != nil {
return err
}
} }
return scanner.Err() return scanner.Err()
}). }).
+1 -6
View File
@@ -421,12 +421,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo *
err = cmdApply. err = cmdApply.
WithDir(tmpBasePath). WithDir(tmpBasePath).
WithStderrReader(&stderrReader). WithStderrReader(&stderrReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
defer func() {
// Close the reader on return to terminate the git command if necessary
_ = stderrReader.Close()
}()
const prefix = "error: patch failed:" const prefix = "error: patch failed:"
const errorPrefix = "error: " const errorPrefix = "error: "
const threewayFailed = "Failed to perform three-way merge..." const threewayFailed = "Failed to perform three-way merge..."
+4 -19
View File
@@ -9,7 +9,6 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"os"
"strconv" "strconv"
"strings" "strings"
@@ -60,25 +59,11 @@ func readUnmergedLsFileLines(ctx context.Context, tmpBasePath string, outputChan
close(outputChan) close(outputChan)
}() }()
lsFilesReader, lsFilesWriter, err := os.Pipe() var lsFilesReader io.ReadCloser
if err != nil { err := gitcmd.NewCommand("ls-files", "-u", "-z").
log.Error("Unable to open stderr pipe: %v", err)
outputChan <- &lsFileLine{err: fmt.Errorf("unable to open stderr pipe: %w", err)}
return
}
defer func() {
_ = lsFilesWriter.Close()
_ = lsFilesReader.Close()
}()
err = gitcmd.NewCommand("ls-files", "-u", "-z").
WithDir(tmpBasePath). WithDir(tmpBasePath).
WithStdout(lsFilesWriter). WithStdoutReader(&lsFilesReader).
WithPipelineFunc(func(_ context.Context, _ context.CancelFunc) error { WithPipelineFunc(func(_ gitcmd.Context) error {
_ = lsFilesWriter.Close()
defer func() {
_ = lsFilesReader.Close()
}()
bufferedReader := bufio.NewReader(lsFilesReader) bufferedReader := bufio.NewReader(lsFilesReader)
for { for {
+3 -11
View File
@@ -8,7 +8,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"os"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@@ -526,18 +525,11 @@ func checkIfPRContentChanged(ctx context.Context, pr *issues_model.PullRequest,
} }
cmd := gitcmd.NewCommand("diff", "--name-only", "-z").AddDynamicArguments(newCommitID, oldCommitID, mergeBase) cmd := gitcmd.NewCommand("diff", "--name-only", "-z").AddDynamicArguments(newCommitID, oldCommitID, mergeBase)
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return false, mergeBase, fmt.Errorf("unable to open pipe for to run diff: %w", err)
}
var stdoutReader io.ReadCloser
if err := cmd.WithDir(prCtx.tmpBasePath). if err := cmd.WithDir(prCtx.tmpBasePath).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
defer func() {
_ = stdoutReader.Close()
}()
return util.IsEmptyReader(stdoutReader) return util.IsEmptyReader(stdoutReader)
}). }).
RunWithStderr(ctx); err != nil { RunWithStderr(ctx); err != nil {
+12 -4
View File
@@ -9,7 +9,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"regexp"
"strings" "strings"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
@@ -17,6 +16,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo" repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/optional"
@@ -26,7 +26,15 @@ import (
notify_service "code.gitea.io/gitea/services/notify" notify_service "code.gitea.io/gitea/services/notify"
) )
var notEnoughLines = regexp.MustCompile(`fatal: file .* has only \d+ lines?`) func isErrBlameNotFoundOrNotEnoughLines(err error) bool {
stdErr, ok := gitcmd.ErrorAsStderr(err)
if !ok {
return false
}
notFound := strings.HasPrefix(stdErr, "fatal: no such path")
notEnoughLines := strings.HasPrefix(stdErr, "fatal: file ") && strings.Contains(stdErr, " has only ") && strings.Contains(stdErr, " lines?")
return notFound || notEnoughLines
}
// ErrDismissRequestOnClosedPR represents an error when an user tries to dismiss a review associated to a closed or merged PR. // ErrDismissRequestOnClosedPR represents an error when an user tries to dismiss a review associated to a closed or merged PR.
type ErrDismissRequestOnClosedPR struct{} type ErrDismissRequestOnClosedPR struct{}
@@ -67,7 +75,7 @@ func lineBlame(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Re
func checkInvalidation(ctx context.Context, c *issues_model.Comment, repo *repo_model.Repository, gitRepo *git.Repository, branch string) error { func checkInvalidation(ctx context.Context, c *issues_model.Comment, repo *repo_model.Repository, gitRepo *git.Repository, branch string) error {
// FIXME differentiate between previous and proposed line // FIXME differentiate between previous and proposed line
commit, err := lineBlame(ctx, repo, gitRepo, branch, c.TreePath, uint(c.UnsignedLine())) commit, err := lineBlame(ctx, repo, gitRepo, branch, c.TreePath, uint(c.UnsignedLine()))
if err != nil && (strings.Contains(err.Error(), "fatal: no such path") || notEnoughLines.MatchString(err.Error())) { if isErrBlameNotFoundOrNotEnoughLines(err) {
c.Invalidated = true c.Invalidated = true
return issues_model.UpdateCommentInvalidate(ctx, c) return issues_model.UpdateCommentInvalidate(ctx, c)
} }
@@ -251,7 +259,7 @@ func createCodeComment(ctx context.Context, doer *user_model.User, repo *repo_mo
commit, err := lineBlame(ctx, pr.BaseRepo, gitRepo, head, treePath, uint(line)) commit, err := lineBlame(ctx, pr.BaseRepo, gitRepo, head, treePath, uint(line))
if err == nil { if err == nil {
commitID = commit.ID.String() commitID = commit.ID.String()
} else if !(strings.Contains(err.Error(), "exit status 128 - fatal: no such path") || notEnoughLines.MatchString(err.Error())) { } else if !isErrBlameNotFoundOrNotEnoughLines(err) {
return nil, fmt.Errorf("LineBlame[%s, %s, %s, %d]: %w", pr.GetGitHeadRefName(), gitRepo.Path, treePath, line, err) return nil, fmt.Errorf("LineBlame[%s, %s, %s, %d]: %w", pr.GetGitHeadRefName(), gitRepo.Path, treePath, line, err)
} }
} }
+4 -13
View File
@@ -8,7 +8,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"os" "io"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -117,24 +117,16 @@ func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int
if err != nil { if err != nil {
return nil, err return nil, err
} }
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, err
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
gitCmd := gitcmd.NewCommand("log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") gitCmd := gitcmd.NewCommand("log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse")
// AddOptionFormat("--max-count=%d", limit) // AddOptionFormat("--max-count=%d", limit)
gitCmd.AddDynamicArguments(baseCommit.ID.String()) gitCmd.AddDynamicArguments(baseCommit.ID.String())
var stdoutReader io.ReadCloser
var extendedCommitStats []*ExtendedCommitStats var extendedCommitStats []*ExtendedCommitStats
err = gitCmd.WithDir(repo.Path). err = gitCmd.WithDir(repo.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
scanner := bufio.NewScanner(stdoutReader) scanner := bufio.NewScanner(stdoutReader)
for scanner.Scan() { for scanner.Scan() {
@@ -186,7 +178,6 @@ func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int
} }
extendedCommitStats = append(extendedCommitStats, res) extendedCommitStats = append(extendedCommitStats, res)
} }
_ = stdoutReader.Close()
return nil return nil
}). }).
RunWithStderr(repo.Ctx) RunWithStderr(repo.Ctx)
+5 -15
View File
@@ -362,25 +362,15 @@ func (t *TemporaryUploadRepository) Push(ctx context.Context, doer *user_model.U
// DiffIndex returns a Diff of the current index to the head // DiffIndex returns a Diff of the current index to the head
func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Diff, error) { func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Diff, error) {
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, fmt.Errorf("unable to open stdout pipe: %w", err)
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
var diff *gitdiff.Diff var diff *gitdiff.Diff
err = gitcmd.NewCommand("diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). var stdoutReader io.ReadCloser
err := gitcmd.NewCommand("diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD").
WithTimeout(30 * time.Second). WithTimeout(30 * time.Second).
WithDir(t.basePath). WithDir(t.basePath).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close()
defer cancel()
var diffErr error var diffErr error
diff, diffErr = gitdiff.ParsePatch(ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") diff, diffErr = gitdiff.ParsePatch(ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "")
_ = stdoutReader.Close()
if diffErr != nil { if diffErr != nil {
// if the diffErr is not nil, it will be returned as the error of "Run()" // if the diffErr is not nil, it will be returned as the error of "Run()"
return fmt.Errorf("ParsePatch: %w", diffErr) return fmt.Errorf("ParsePatch: %w", diffErr)
@@ -388,7 +378,7 @@ func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Dif
return nil return nil
}). }).
RunWithStderr(ctx) RunWithStderr(ctx)
if err != nil && !git.IsErrCanceledOrKilled(err) { if err != nil && !gitcmd.IsErrorCanceledOrKilled(err) {
return nil, fmt.Errorf("unable to run diff-index pipeline in temporary repo: %w", err) return nil, fmt.Errorf("unable to run diff-index pipeline in temporary repo: %w", err)
} }
+7 -16
View File
@@ -6,8 +6,7 @@ package gitgraph
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"context" "io"
"os"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/git/gitcmd"
@@ -44,20 +43,14 @@ func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bo
} }
graph := NewGraph() graph := NewGraph()
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, err
}
commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1) commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1)
scanner := bufio.NewScanner(stdoutReader) var stdoutReader io.ReadCloser
if err := graphCmd. if err := graphCmd.
WithDir(r.Path). WithDir(r.Path).
WithStdout(stdoutWriter). WithStdoutReader(&stdoutReader).
WithPipelineFunc(func(ctx context.Context, cancel context.CancelFunc) error { WithPipelineFunc(func(ctx gitcmd.Context) error {
_ = stdoutWriter.Close() scanner := bufio.NewScanner(stdoutReader)
defer stdoutReader.Close()
parser := &Parser{} parser := &Parser{}
parser.firstInUse = -1 parser.firstInUse = -1
parser.maxAllowedColors = maxAllowedColors parser.maxAllowedColors = maxAllowedColors
@@ -89,8 +82,7 @@ func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bo
line := scanner.Bytes() line := scanner.Bytes()
if bytes.IndexByte(line, '*') >= 0 { if bytes.IndexByte(line, '*') >= 0 {
if err := parser.AddLineToGraph(graph, row, line); err != nil { if err := parser.AddLineToGraph(graph, row, line); err != nil {
cancel() return ctx.CancelWithCause(err)
return err
} }
break break
} }
@@ -101,8 +93,7 @@ func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bo
row++ row++
line := scanner.Bytes() line := scanner.Bytes()
if err := parser.AddLineToGraph(graph, row, line); err != nil { if err := parser.AddLineToGraph(graph, row, line); err != nil {
cancel() return ctx.CancelWithCause(err)
return err
} }
} }
return scanner.Err() return scanner.Err()
-4
View File
@@ -307,10 +307,6 @@
<dd>{{.Git.Timeout.Migrate}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd> <dd>{{.Git.Timeout.Migrate}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.git_mirror_timeout"}}</dt> <dt>{{ctx.Locale.Tr "admin.config.git_mirror_timeout"}}</dt>
<dd>{{.Git.Timeout.Mirror}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd> <dd>{{.Git.Timeout.Mirror}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.git_clone_timeout"}}</dt>
<dd>{{.Git.Timeout.Clone}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.git_pull_timeout"}}</dt>
<dd>{{.Git.Timeout.Pull}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.git_gc_timeout"}}</dt> <dt>{{ctx.Locale.Tr "admin.config.git_gc_timeout"}}</dt>
<dd>{{.Git.Timeout.GC}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd> <dd>{{.Git.Timeout.GC}} {{ctx.Locale.Tr "tool.raw_seconds"}}</dd>
</dl> </dl>