Skip to content

Commit a09f536

Browse files
imeoerimeoer
committed
nydusify: refactor check subcommand
- allow either the source or target to be an OCI or nydus image; - improve output directory structure and log format; Signed-off-by: Yan Song <[email protected]>
1 parent 375f55f commit a09f536

File tree

11 files changed

+539
-376
lines changed

11 files changed

+539
-376
lines changed

‎contrib/nydusify/cmd/nydusify.go

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -559,19 +559,39 @@ func main() {
559559
},
560560

561561
&cli.StringFlag{
562-
Name: "backend-type",
562+
Name: "source-backend-type",
563563
Value: "",
564-
Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3'",
564+
Usage: "Type of storage backend, possible values: 'oss', 's3'",
565565
EnvVars: []string{"BACKEND_TYPE"},
566566
},
567567
&cli.StringFlag{
568-
Name: "backend-config",
568+
Name: "source-backend-config",
569569
Value: "",
570-
Usage: "Json string for storage backend configuration",
570+
Usage: "Json configuration string for storage backend",
571571
EnvVars: []string{"BACKEND_CONFIG"},
572572
},
573573
&cli.PathFlag{
574-
Name: "backend-config-file",
574+
Name: "source-backend-config-file",
575+
Value: "",
576+
TakesFile: true,
577+
Usage: "Json configuration file for storage backend",
578+
EnvVars: []string{"BACKEND_CONFIG_FILE"},
579+
},
580+
581+
&cli.StringFlag{
582+
Name: "target-backend-type",
583+
Value: "",
584+
Usage: "Type of storage backend, possible values: 'oss', 's3'",
585+
EnvVars: []string{"BACKEND_TYPE"},
586+
},
587+
&cli.StringFlag{
588+
Name: "target-backend-config",
589+
Value: "",
590+
Usage: "Json configuration string for storage backend",
591+
EnvVars: []string{"BACKEND_CONFIG"},
592+
},
593+
&cli.PathFlag{
594+
Name: "target-backend-config-file",
575595
Value: "",
576596
TakesFile: true,
577597
Usage: "Json configuration file for storage backend",
@@ -612,7 +632,12 @@ func main() {
612632
Action: func(c *cli.Context) error {
613633
setupLogLevel(c)
614634

615-
backendType, backendConfig, err := getBackendConfig(c, "", false)
635+
sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
636+
if err != nil {
637+
return err
638+
}
639+
640+
targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false)
616641
if err != nil {
617642
return err
618643
}
@@ -623,16 +648,20 @@ func main() {
623648
}
624649

625650
checker, err := checker.New(checker.Opt{
626-
WorkDir: c.String("work-dir"),
627-
Source: c.String("source"),
628-
Target: c.String("target"),
651+
WorkDir: c.String("work-dir"),
652+
653+
Source: c.String("source"),
654+
Target: c.String("target"),
655+
SourceInsecure: c.Bool("source-insecure"),
656+
TargetInsecure: c.Bool("target-insecure"),
657+
SourceBackendType: sourceBackendType,
658+
SourceBackendConfig: sourceBackendConfig,
659+
TargetBackendType: targetBackendType,
660+
TargetBackendConfig: targetBackendConfig,
661+
629662
MultiPlatform: c.Bool("multi-platform"),
630-
SourceInsecure: c.Bool("source-insecure"),
631-
TargetInsecure: c.Bool("target-insecure"),
632663
NydusImagePath: c.String("nydus-image"),
633664
NydusdPath: c.String("nydusd"),
634-
BackendType: backendType,
635-
BackendConfig: backendConfig,
636665
ExpectedArch: arch,
637666
})
638667
if err != nil {

‎contrib/nydusify/pkg/checker/checker.go

Lines changed: 55 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -13,43 +13,44 @@ import (
1313
"github.com/sirupsen/logrus"
1414

1515
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule"
16-
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool"
1716
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
1817
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
19-
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote"
2018
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
2119
)
2220

2321
// Opt defines Checker options.
24-
// Note: target is the Nydus image reference.
22+
// Note: target is the nydus image reference.
2523
type Opt struct {
26-
WorkDir string
27-
Source string
28-
Target string
29-
SourceInsecure bool
30-
TargetInsecure bool
24+
WorkDir string
25+
26+
Source string
27+
Target string
28+
SourceInsecure bool
29+
TargetInsecure bool
30+
SourceBackendType string
31+
SourceBackendConfig string
32+
TargetBackendType string
33+
TargetBackendConfig string
34+
3135
MultiPlatform bool
3236
NydusImagePath string
3337
NydusdPath string
34-
BackendType string
35-
BackendConfig string
3638
ExpectedArch string
3739
}
3840

39-
// Checker validates Nydus image manifest, bootstrap and mounts filesystem
40-
// by Nydusd to compare file metadata and data with OCI image.
41+
// Checker validates nydus image manifest, bootstrap and mounts filesystem
42+
// by nydusd to compare file metadata and data between OCI / nydus image.
4143
type Checker struct {
4244
Opt
4345
sourceParser *parser.Parser
4446
targetParser *parser.Parser
4547
}
4648

47-
// New creates Checker instance, target is the Nydus image reference.
49+
// New creates Checker instance, target is the nydus image reference.
4850
func New(opt Opt) (*Checker, error) {
49-
// TODO: support source and target resolver
5051
targetRemote, err := provider.DefaultRemote(opt.Target, opt.TargetInsecure)
5152
if err != nil {
52-
return nil, errors.Wrap(err, "Init target image parser")
53+
return nil, errors.Wrap(err, "init target image parser")
5354
}
5455
targetParser, err := parser.New(targetRemote, opt.ExpectedArch)
5556
if err != nil {
@@ -63,7 +64,7 @@ func New(opt Opt) (*Checker, error) {
6364
return nil, errors.Wrap(err, "Init source image parser")
6465
}
6566
sourceParser, err = parser.New(sourceRemote, opt.ExpectedArch)
66-
if sourceParser == nil {
67+
if err != nil {
6768
return nil, errors.Wrap(err, "failed to create parser")
6869
}
6970
}
@@ -77,7 +78,7 @@ func New(opt Opt) (*Checker, error) {
7778
return checker, nil
7879
}
7980

80-
// Check checks Nydus image, and outputs image information to work
81+
// Check checks nydus image, and outputs image information to work
8182
// directory, the check workflow is composed of various rules.
8283
func (checker *Checker) Check(ctx context.Context) error {
8384
if err := checker.check(ctx); err != nil {
@@ -93,12 +94,13 @@ func (checker *Checker) Check(ctx context.Context) error {
9394
return nil
9495
}
9596

96-
// Check checks Nydus image, and outputs image information to work
97+
// Check checks nydus image, and outputs image information to work
9798
// directory, the check workflow is composed of various rules.
9899
func (checker *Checker) check(ctx context.Context) error {
100+
logrus.WithField("image", checker.targetParser.Remote.Ref).Infof("parsing image")
99101
targetParsed, err := checker.targetParser.Parse(ctx)
100102
if err != nil {
101-
return errors.Wrap(err, "parse Nydus image")
103+
return errors.Wrap(err, "parse nydus image")
102104
}
103105

104106
var sourceParsed *parser.Parsed
@@ -107,89 +109,66 @@ func (checker *Checker) check(ctx context.Context) error {
107109
if err != nil {
108110
return errors.Wrap(err, "parse source image")
109111
}
110-
} else {
111-
sourceParsed = targetParsed
112112
}
113113

114114
if err := os.RemoveAll(checker.WorkDir); err != nil {
115115
return errors.Wrap(err, "clean up work directory")
116116
}
117117

118-
if err := os.MkdirAll(filepath.Join(checker.WorkDir, "fs"), 0755); err != nil {
119-
return errors.Wrap(err, "create work directory")
120-
}
121-
122-
if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil {
123-
return errors.Wrap(err, "output image information")
124-
}
125-
126-
mode := "direct"
127-
digestValidate := false
128-
if targetParsed.NydusImage != nil {
129-
nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest)
130-
if nydusManifest != nil {
131-
v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5)
132-
if v == utils.V5 {
133-
// Digest validate is not currently supported for v6,
134-
// but v5 supports it. In order to make the check more sufficient,
135-
// this validate needs to be turned on for v5.
136-
digestValidate = true
137-
}
118+
if sourceParsed != nil {
119+
if err := checker.Output(ctx, sourceParsed, filepath.Join(checker.WorkDir, "source")); err != nil {
120+
return errors.Wrapf(err, "output image information: %s", sourceParsed.Remote.Ref)
138121
}
139122
}
140123

141-
var sourceRemote *remote.Remote
142-
if checker.sourceParser != nil {
143-
sourceRemote = checker.sourceParser.Remote
124+
if targetParsed != nil {
125+
if err := checker.Output(ctx, targetParsed, filepath.Join(checker.WorkDir, "target")); err != nil {
126+
return errors.Wrapf(err, "output image information: %s", targetParsed.Remote.Ref)
127+
}
144128
}
145129

146130
rules := []rule.Rule{
147131
&rule.ManifestRule{
148-
SourceParsed: sourceParsed,
149-
TargetParsed: targetParsed,
150-
MultiPlatform: checker.MultiPlatform,
151-
BackendType: checker.BackendType,
152-
ExpectedArch: checker.ExpectedArch,
132+
SourceParsed: sourceParsed,
133+
TargetParsed: targetParsed,
153134
},
154135
&rule.BootstrapRule{
155-
Parsed: targetParsed,
156-
NydusImagePath: checker.NydusImagePath,
157-
BackendType: checker.BackendType,
158-
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
159-
DebugOutputPath: filepath.Join(checker.WorkDir, "nydus_bootstrap_debug.json"),
136+
WorkDir: checker.WorkDir,
137+
NydusImagePath: checker.NydusImagePath,
138+
139+
SourceParsed: sourceParsed,
140+
TargetParsed: targetParsed,
141+
SourceBackendType: checker.SourceBackendType,
142+
SourceBackendConfig: checker.SourceBackendConfig,
143+
TargetBackendType: checker.TargetBackendType,
144+
TargetBackendConfig: checker.TargetBackendConfig,
160145
},
161146
&rule.FilesystemRule{
162-
Source: checker.Source,
163-
SourceMountPath: filepath.Join(checker.WorkDir, "fs/source_mounted"),
164-
SourceParsed: sourceParsed,
165-
SourcePath: filepath.Join(checker.WorkDir, "fs/source"),
166-
SourceRemote: sourceRemote,
167-
Target: checker.Target,
168-
TargetInsecure: checker.TargetInsecure,
169-
PlainHTTP: checker.targetParser.Remote.IsWithHTTP(),
170-
NydusdConfig: tool.NydusdConfig{
171-
EnablePrefetch: true,
172-
NydusdPath: checker.NydusdPath,
173-
BackendType: checker.BackendType,
174-
BackendConfig: checker.BackendConfig,
175-
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
176-
ConfigPath: filepath.Join(checker.WorkDir, "fs/nydusd_config.json"),
177-
BlobCacheDir: filepath.Join(checker.WorkDir, "fs/nydus_blobs"),
178-
MountPath: filepath.Join(checker.WorkDir, "fs/nydus_mounted"),
179-
APISockPath: filepath.Join(checker.WorkDir, "fs/nydus_api.sock"),
180-
Mode: mode,
181-
DigestValidate: digestValidate,
147+
WorkDir: checker.WorkDir,
148+
NydusdPath: checker.NydusdPath,
149+
150+
SourceImage: &rule.Image{
151+
Parsed: sourceParsed,
152+
Insecure: checker.SourceInsecure,
153+
},
154+
TargetImage: &rule.Image{
155+
Parsed: targetParsed,
156+
Insecure: checker.TargetInsecure,
182157
},
158+
SourceBackendType: checker.SourceBackendType,
159+
SourceBackendConfig: checker.SourceBackendConfig,
160+
TargetBackendType: checker.TargetBackendType,
161+
TargetBackendConfig: checker.TargetBackendConfig,
183162
},
184163
}
185164

186165
for _, rule := range rules {
187166
if err := rule.Validate(); err != nil {
188-
return errors.Wrapf(err, "validate rule %s", rule.Name())
167+
return errors.Wrapf(err, "validate %s failed", rule.Name())
189168
}
190169
}
191170

192-
logrus.Infof("Verified Nydus image %s", checker.targetParser.Remote.Ref)
171+
logrus.Info("verified image")
193172

194173
return nil
195174
}

0 commit comments

Comments
 (0)