feat: allow 'graph' to be generated without 'target'

This switches off dependency resolution mode and dumps all the stages
(targets) as a graph.

While working on this feature, I discovered two cases when parts of the
build graph were duplicated: first when packages were resolved, leading
to duplicate nodes in the graph, second when LLB is generated. Buildkit
seems to handle correctly LLB de-duplication, but anyway it's better to
skip duplicate entries.

Fixes #22

As examples, see siderolabs/tools#73 and siderolabs/pkgs#46

Signed-off-by: Andrey Smirnov <[email protected]>

Author: smira

cmd/graph.go

@@ -24,20 +24,30 @@ starting from target to all the dependencies.
 			Root:    pkgRoot,
 			Context: options.GetVariables(),
 		}
+
 		packages, err := solver.NewPackages(&loader)
 		if err != nil {
 			log.Fatal(err)
 		}
-		graph, err := packages.Resolve(options.Target)
-		if err != nil {
-			log.Fatal(err)
+
+		var packageSet solver.PackageSet
+
+		if options.Target != "" {
+			graph, err := packages.Resolve(options.Target)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			packageSet = graph.ToSet()
+		} else {
+			packageSet = packages.ToSet()
 		}
-		graph.DumpDot(os.Stdout)
+
+		packageSet.DumpDot(os.Stdout)
 	},
 }
 
 func init() {
-	graphCmd.Flags().StringVarP(&options.Target, "target", "t", "", "Target image to build")
-	graphCmd.MarkFlagRequired("target") //nolint: errcheck
+	graphCmd.Flags().StringVarP(&options.Target, "target", "t", "", "Target image to graph, if not set - graph all stages")
 	rootCmd.AddCommand(graphCmd)
 }

internal/pkg/convert/graph.go

@@ -25,13 +25,16 @@ type GraphLLB struct {
 	BaseImages   map[v1alpha2.Variant]llb.State
 	Checksummer  llb.State
 	LocalContext llb.State
+
+	cache map[*solver.PackageNode]llb.State
 }
 
 // NewGraphLLB creates new GraphLLB and initializes shared images.
 func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *GraphLLB {
 	result := &GraphLLB{
 		PackageGraph: graph,
 		Options:      options,
+		cache:        make(map[*solver.PackageNode]llb.State),
 	}
 
 	result.buildBaseImages()

internal/pkg/convert/node.go

@@ -6,6 +6,7 @@ package convert
 
 import (
 	"fmt"
+	"log"
 	"path/filepath"
 	"sort"
 
@@ -214,6 +215,11 @@ func (node *NodeLLB) finalize(root llb.State) llb.State {
 func (node *NodeLLB) Build() (llb.State, error) {
 	var err error
 
+	if state, ok := node.Graph.cache[node.PackageNode]; ok {
+		log.Printf("cached node %s", node.Name)
+		return state, nil
+	}
+
 	root := node.base()
 	root = node.install(root)
 	root = node.context(root)
@@ -229,5 +235,7 @@ func (node *NodeLLB) Build() (llb.State, error) {
 
 	root = node.finalize(root)
 
+	node.Graph.cache[node.PackageNode] = root
+
 	return root, nil
 }

internal/pkg/solver/graph.go

@@ -5,8 +5,6 @@
 package solver
 
 import (
-	"io"
-
 	"github.com/emicklei/dot"
 
 	"github.com/talos-systems/bldr/internal/pkg/types/v1alpha2"
@@ -23,11 +21,9 @@ type PackageNode struct {
 func (node *PackageNode) DumpDot(g *dot.Graph) dot.Node {
 	n := g.Node(node.Name)
 
-	for _, dep := range node.DependsOn {
-		depNode := dep.DumpDot(g)
-		if len(depNode.EdgesTo(n)) == 0 {
-			depNode.Edge(n)
-		}
+	for _, dep := range node.Pkg.InternalDependencies() {
+		depNode := g.Node(dep)
+		depNode.Edge(n)
 	}
 
 	for _, dep := range node.Pkg.ExternalDependencies() {
@@ -36,9 +32,7 @@ func (node *PackageNode) DumpDot(g *dot.Graph) dot.Node {
 		imageNode.Attr("fillcolor", "lemonchiffon")
 		imageNode.Attr("style", "filled")
 
-		if len(imageNode.EdgesTo(n)) == 0 {
-			imageNode.Edge(n)
-		}
+		imageNode.Edge(n)
 	}
 
 	return n
@@ -49,10 +43,22 @@ type PackageGraph struct {
 	Root *PackageNode
 }
 
-// DumpDot dumps whole graph in dot format
-func (graph *PackageGraph) DumpDot(w io.Writer) {
-	g := dot.NewGraph(dot.Directed)
-	graph.Root.DumpDot(g)
+func (graph *PackageGraph) flatten(set PackageSet, node *PackageNode, skip map[*PackageNode]struct{}) PackageSet {
+	if _, exists := skip[node]; exists {
+		return set
+	}
+
+	set = append(set, node)
+	skip[node] = struct{}{}
+
+	for _, dep := range node.DependsOn {
+		set = graph.flatten(set, dep, skip)
+	}
+
+	return set
+}
 
-	g.Write(w)
+// ToSet converts graph to set of nodes
+func (graph *PackageGraph) ToSet() PackageSet {
+	return graph.flatten(nil, graph.Root, make(map[*PackageNode]struct{}))
 }

internal/pkg/solver/packages.go

@@ -39,7 +39,11 @@ func NewPackages(loader PackageLoader) (*Packages, error) {
 	return result, nil
 }
 
-func (pkgs *Packages) resolve(name string, path []string) (*PackageNode, error) {
+func (pkgs *Packages) resolve(name string, path []string, cache map[string]*PackageNode) (*PackageNode, error) {
+	if node := cache[name]; node != nil {
+		return node, nil
+	}
+
 	pkg := pkgs.packages[name]
 	if pkg == nil {
 		return nil, fmt.Errorf("package %q not defined", name)
@@ -60,23 +64,37 @@ func (pkgs *Packages) resolve(name string, path []string) (*PackageNode, error)
 
 	deps := pkg.InternalDependencies()
 	for _, dep := range deps {
-		depPkg, err := pkgs.resolve(dep, path)
+		depPkg, err := pkgs.resolve(dep, path, cache)
 		if err != nil {
 			return nil, fmt.Errorf("error resolving dependency %q of %q: %w", dep, name, err)
 		}
 
 		node.DependsOn = append(node.DependsOn, depPkg)
 	}
 
+	cache[name] = node
+
 	return node, nil
 }
 
 // Resolve trims down the package tree to have only deps of the target
 func (pkgs *Packages) Resolve(target string) (*PackageGraph, error) {
-	root, err := pkgs.resolve(target, nil)
+	root, err := pkgs.resolve(target, nil, make(map[string]*PackageNode))
 	if err != nil {
 		return nil, err
 	}
 
 	return &PackageGraph{root}, nil
 }
+
+// ToSet converts to set of package nodes
+func (pkgs *Packages) ToSet() (set PackageSet) {
+	for name, pkg := range pkgs.packages {
+		set = append(set, &PackageNode{
+			Name: name,
+			Pkg:  pkg,
+		})
+	}
+
+	return
+}

internal/pkg/solver/set.go

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package solver
+
+import (
+	"io"
+
+	"github.com/emicklei/dot"
+)
+
+// PackageSet is a list of PackageNodes
+type PackageSet []*PackageNode
+
+// DumpDot dumps nodes and deps in dot format
+func (set PackageSet) DumpDot(w io.Writer) {
+	g := dot.NewGraph(dot.Directed)
+
+	for _, node := range set {
+		node.DumpDot(g)
+	}
+
+	g.Write(w)
+}