nydusify(feat): support for specifing log file and concurrently processing external model manifests

Signed-off-by: Yang Kaiyong <[email protected]>

Author: BraveY

contrib/nydusify/cmd/nydusify.go

@@ -193,22 +193,7 @@ func main() {
 	}
 
 	// global options
-	app.Flags = []cli.Flag{
-		&cli.BoolFlag{
-			Name:     "debug",
-			Aliases:  []string{"D"},
-			Required: false,
-			Value:    false,
-			Usage:    "Enable debug log level, overwrites the 'log-level' option",
-			EnvVars:  []string{"DEBUG_LOG_LEVEL"}},
-		&cli.StringFlag{
-			Name:    "log-level",
-			Aliases: []string{"l"},
-			Value:   "info",
-			Usage:   "Set log level (panic, fatal, error, warn, info, debug, trace)",
-			EnvVars: []string{"LOG_LEVEL"},
-		},
-	}
+	app.Flags = getGlobalFlags()
 
 	app.Commands = []*cli.Command{
 		{
@@ -1439,4 +1424,38 @@ func setupLogLevel(c *cli.Context) {
 	}
 
 	logrus.SetLevel(logLevel)
+
+	if c.String("log-file") != "" {
+		f, err := os.OpenFile(c.String("log-file"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+		if err != nil {
+			logrus.Errorf("failed to open log file: %+v", err)
+			return
+		}
+		logrus.SetOutput(f)
+	}
+}
+
+func getGlobalFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:     "debug",
+			Aliases:  []string{"D"},
+			Required: false,
+			Value:    false,
+			Usage:    "Enable debug log level, overwrites the 'log-level' option",
+			EnvVars:  []string{"DEBUG_LOG_LEVEL"}},
+		&cli.StringFlag{
+			Name:    "log-level",
+			Aliases: []string{"l"},
+			Value:   "info",
+			Usage:   "Set log level (panic, fatal, error, warn, info, debug, trace)",
+			EnvVars: []string{"LOG_LEVEL"},
+		},
+		&cli.StringFlag{
+			Name:     "log-file",
+			Required: false,
+			Usage:    "Write logs to a file",
+			EnvVars:  []string{"LOG_FILE"},
+		},
+	}
 }

contrib/nydusify/cmd/nydusify_test.go

@@ -12,6 +12,9 @@ import (
 	"os"
 	"testing"
 
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
 )
@@ -341,3 +344,50 @@ func TestGetPrefetchPatterns(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "/", patterns)
 }
+
+func TestGetGlobalFlags(t *testing.T) {
+	flags := getGlobalFlags()
+	require.Equal(t, 3, len(flags))
+}
+
+func TestSetupLogLevelWithLogFile(t *testing.T) {
+	logFilePath := "test_log_file.log"
+	defer os.Remove(logFilePath)
+
+	c := &cli.Context{}
+
+	patches := gomonkey.ApplyMethodSeq(c, "String", []gomonkey.OutputCell{
+		{Values: []interface{}{"info"}, Times: 1},
+		{Values: []interface{}{"test_log_file.log"}, Times: 2},
+	})
+	defer patches.Reset()
+	setupLogLevel(c)
+
+	file, err := os.Open(logFilePath)
+	assert.NoError(t, err)
+	assert.NotNil(t, file)
+	file.Close()
+
+	logrusOutput := logrus.StandardLogger().Out
+	assert.NotNil(t, logrusOutput)
+
+	logrus.Info("This is a test log message")
+	content, err := os.ReadFile(logFilePath)
+	assert.NoError(t, err)
+	assert.Contains(t, string(content), "This is a test log message")
+}
+
+func TestSetupLogLevelWithInvalidLogFile(t *testing.T) {
+
+	c := &cli.Context{}
+
+	patches := gomonkey.ApplyMethodSeq(c, "String", []gomonkey.OutputCell{
+		{Values: []interface{}{"info"}, Times: 1},
+		{Values: []interface{}{"test/test_log_file.log"}, Times: 2},
+	})
+	defer patches.Reset()
+	setupLogLevel(c)
+
+	logrusOutput := logrus.StandardLogger().Out
+	assert.NotNil(t, logrusOutput)
+}

contrib/nydusify/go.mod

@@ -39,7 +39,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
-	github.com/BraveY/snapshotter-converter v0.0.6 // indirect
+	github.com/BraveY/snapshotter-converter v0.0.5 // indirect
 	github.com/CloudNativeAI/model-spec v0.0.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect

contrib/nydusify/pkg/converter/converter.go

@@ -262,7 +262,6 @@ func convertModelArtifact(ctx context.Context, opt Opt) error {
 		ChunkSize:      opt.ChunkSize,
 		FromDir:        contextDir,
 		AttributesPath: attributesPath,
-		Crc32:          true,
 	}
 	_, externalBlobDigest, err := packWithAttributes(ctx, packOption, tmpDir)
 	if err != nil {

contrib/nydusify/pkg/external/modctl/remote.go

@@ -10,11 +10,13 @@ import (
 	"io"
 	"os"
 	"strconv"
+	"sync"
 
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/snapshotter/external/backend"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
 	pkgPvd "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
@@ -80,14 +82,31 @@ func initRemoteHandler(handler *RemoteHandler) error {
 }
 
 func (handler *RemoteHandler) Handle(ctx context.Context) (*backend.Backend, []backend.FileAttribute, error) {
-	var fileAttrs []backend.FileAttribute
+	var (
+		fileAttrs []backend.FileAttribute
+		mu        sync.Mutex
+		eg        *errgroup.Group
+	)
+	eg, ctx = errgroup.WithContext(ctx)
+	eg.SetLimit(5)
+
 	for idx, layer := range handler.manifest.Layers {
-		fa, err := handler.handle(ctx, layer, int32(idx))
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "handle layer failed")
-		}
-		fileAttrs = append(fileAttrs, fa...)
+		eg.Go(func() error {
+			fa, err := handler.handle(ctx, layer, int32(idx))
+			if err != nil {
+				return err
+			}
+			mu.Lock()
+			fileAttrs = append(fileAttrs, fa...)
+			mu.Unlock()
+			return nil
+		})
 	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, nil, errors.Wrap(err, "wait for handle failed")
+	}
+
 	bkd, err := handler.backend()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "get backend failed")

contrib/nydusify/pkg/snapshotter/external/external.go

@@ -95,7 +95,7 @@ func RemoteHandle(ctx context.Context, opts Options) error {
 		attributes = append(attributes, Attribute{
 			Pattern: p,
 		})
-		logrus.Infof("file attr: %s, file_mode: %o", p, file.Mode)
+		logrus.Debugf("file attr: %s, file_mode: %o", p, file.Mode)
 	}
 
 	backendBytes, err := json.MarshalIndent(bkd, "", "  ")