nydusify(refactor): handle layer with retry

Signed-off-by: Yang Kaiyong <[email protected]>

Author: BraveY

contrib/nydusify/pkg/copier/copier.go

@@ -67,23 +67,6 @@ type output struct {
 	Blobs []string
 }
 
-func withRetry(handle func() error, total int) error {
-	for {
-		total--
-		err := handle()
-		if err == nil {
-			return nil
-		}
-
-		if total > 0 && !errors.Is(err, context.Canceled) {
-			logrus.WithError(err).Warnf("retry (remain %d times)", total)
-			continue
-		}
-
-		return err
-	}
-}
-
 func hosts(opt Opt) remote.HostFunc {
 	maps := map[string]bool{
 		opt.Source: opt.SourceInsecure,
@@ -189,7 +172,7 @@ func pushBlobFromBackend(
 					},
 				}
 
-				if err := withRetry(func() error {
+				if err := nydusifyUtils.RetryWithAttempts(func() error {
 					pusher, err := getPusherInChunked(ctx, pvd, blobDescs[idx], opt)
 					if err != nil {
 						if errdefs.NeedsRetryWithHTTP(err) {

contrib/nydusify/pkg/external/modctl/remote.go

@@ -88,11 +88,16 @@ func (handler *RemoteHandler) Handle(ctx context.Context) (*backend.Backend, []b
 		eg        *errgroup.Group
 	)
 	eg, ctx = errgroup.WithContext(ctx)
-	eg.SetLimit(5)
+	eg.SetLimit(10)
 
 	for idx, layer := range handler.manifest.Layers {
 		eg.Go(func() error {
-			fa, err := handler.handle(ctx, layer, int32(idx))
+			var fa []backend.FileAttribute
+			err := utils.RetryWithAttempts(func() error {
+				_fa, err := handler.handle(ctx, layer, int32(idx))
+				fa = _fa
+				return err
+			}, 5)
 			if err != nil {
 				return err
 			}

contrib/nydusify/pkg/utils/utils.go

@@ -6,6 +6,7 @@ package utils
 
 import (
 	"archive/tar"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -79,6 +80,23 @@ func WithRetry(op func() error) error {
 	return err
 }
 
+func RetryWithAttempts(handle func() error, attempts int) error {
+	for {
+		attempts--
+		err := handle()
+		if err == nil {
+			return nil
+		}
+
+		if attempts > 0 && !errors.Is(err, context.Canceled) {
+			logrus.WithError(err).Warnf("retry (remain %d times)", attempts)
+			continue
+		}
+
+		return err
+	}
+}
+
 func RetryWithHTTP(err error) bool {
 	return err != nil && (errors.Is(err, http.ErrSchemeMismatch) || errors.Is(err, syscall.ECONNREFUSED)) || errdefs.NeedsRetryWithHTTP(err)
 }

contrib/nydusify/pkg/utils/utils_test.go

@@ -8,6 +8,7 @@ package utils
 import (
 	"archive/tar"
 	"compress/gzip"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -273,3 +274,30 @@ func TestGetNydusFsVersionOrDefault(t *testing.T) {
 	fsVersion = GetNydusFsVersionOrDefault(testAnnotations, V5)
 	require.Equal(t, fsVersion, V5)
 }
+
+func TestRetryWithAttempts_SuccessOnFirstAttempt(t *testing.T) {
+	err := RetryWithAttempts(func() error {
+		return nil
+	}, 3)
+	require.NoError(t, err)
+
+	attempts := 0
+	err = RetryWithAttempts(func() error {
+		attempts++
+		if attempts == 1 {
+			return errors.New("first attempt failed")
+		}
+		return nil
+	}, 3)
+	require.NoError(t, err)
+
+	err = RetryWithAttempts(func() error {
+		return errors.New("always fails")
+	}, 3)
+	require.Error(t, err)
+
+	err = RetryWithAttempts(func() error {
+		return context.Canceled
+	}, 3)
+	require.Equal(t, context.Canceled, err)
+}