helper: three attempts with a pause, the Go proxy and the registry both fail transiently under load
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
9dd6e2055f
commit
cdae4bc0c3
2 changed files with 11 additions and 7 deletions
|
|
@ -83,9 +83,10 @@ func buildImage() error {
|
|||
// two names in one build does. Build once per name instead: the later builds are fully
|
||||
// cached and find the blobs already in the registry.
|
||||
// Forgejo also races parallel layer uploads of a brand new image, hence the retry.
|
||||
attempts := 1
|
||||
// Transient failures also come from the Go proxy (504 under load), so the build retries too.
|
||||
attempts := 2
|
||||
if getPushImage() == "true" {
|
||||
attempts = 2
|
||||
attempts = 3
|
||||
}
|
||||
for i, tag := range []string{getImageTag(), getCommitSha()} {
|
||||
metadata := buildMetadataPath
|
||||
|
|
@ -99,15 +100,17 @@ func buildImage() error {
|
|||
args = append(args, strings.Fields(getAdditionalArgs())...)
|
||||
|
||||
fmt.Println("Assembled build command: ", "buildctl", strings.Join(args, " "))
|
||||
if err := withRetry(attempts, func() error { return run(append([]string{"buildctl"}, args...)) }); err != nil {
|
||||
if err := withRetry(attempts, retryDelay, func() error { return run(append([]string{"buildctl"}, args...)) }); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withRetry calls f up to attempts times and returns the last error.
|
||||
func withRetry(attempts int, f func() error) error {
|
||||
const retryDelay = 10 * time.Second
|
||||
|
||||
// withRetry calls f up to attempts times, waiting delay between tries, and returns the last error.
|
||||
func withRetry(attempts int, delay time.Duration, f func() error) error {
|
||||
var err error
|
||||
for i := 1; i <= attempts; i++ {
|
||||
if err = f(); err == nil {
|
||||
|
|
@ -115,6 +118,7 @@ func withRetry(attempts int, f func() error) error {
|
|||
}
|
||||
if i < attempts {
|
||||
fmt.Printf("Attempt %d/%d failed (%v), retrying\n", i, attempts, err)
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ func TestWriteStepOutputs(t *testing.T) {
|
|||
|
||||
func TestWithRetry(t *testing.T) {
|
||||
calls := 0
|
||||
err := withRetry(2, func() error {
|
||||
err := withRetry(2, 0, func() error {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return errors.New("first fails")
|
||||
|
|
@ -146,7 +146,7 @@ func TestWithRetry(t *testing.T) {
|
|||
}
|
||||
|
||||
calls = 0
|
||||
if err := withRetry(2, func() error { calls++; return errors.New("always") }); err == nil || calls != 2 {
|
||||
if err := withRetry(2, 0, func() error { calls++; return errors.New("always") }); err == nil || calls != 2 {
|
||||
t.Errorf("err = %v after %d calls, want error after 2", err, calls)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue