tetragon: fix graceful shutdown and exit code

Right now tetragon always exit with 1, let's fix this and improve
graceful shutdown, so service monitors and container managers...
won't treat Tetragon as it failed.

Some start up logic was re-ordered to improve how we cleanup things.

Also this change will make tetragonExecute() return up to callers,
and allow Tetragon main to exit with 0.

Signed-off-by: Djalal Harouni <[email protected]>

Author: tixxdz

cmd/tetragon/main.go

@@ -119,6 +119,9 @@ func stopProfile() {
 }
 
 func tetragonExecute() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
@@ -143,6 +146,12 @@ func tetragonExecute() error {
 		defaults.NetnsDir = viper.GetString(keyNetnsDir)
 	}
 
+	// Setup file system mounts
+	bpf.CheckOrMountFS("")
+	bpf.CheckOrMountDebugFS()
+	bpf.CheckOrMountCgroup2()
+
+	// Start profilers first as we have to capture them in signal handling
 	if memProfile != "" {
 		log.WithField("file", memProfile).Info("Starting mem profiling")
 	}
@@ -160,25 +169,35 @@ func tetragonExecute() error {
 		log.WithField("file", cpuProfile).Info("Starting cpu profiling")
 	}
 
-	bpf.CheckOrMountFS("")
-	bpf.CheckOrMountDebugFS()
-	bpf.CheckOrMountCgroup2()
-
-	sensors.LogRegisteredSensorsAndProbes()
+	defer stopProfile()
 
+	// Raise memory resource
 	bpf.ConfigureResourceLimits()
+
+	// Get observer bpf maps and programs directory
 	observerDir := getObserverDir()
 	option.Config.BpfDir = observerDir
 	option.Config.MapDir = observerDir
+
+	// Get observer from configFile
 	obs := observer.NewObserver(configFile)
+	defer func() {
+		obs.PrintStats()
+		obs.RemovePrograms()
+	}()
+
+	go func() {
+		s := <-sigs
+		log.Infof("Received signal %s, shutting down...", s)
+		cancel()
+	}()
+
+	sensors.LogRegisteredSensorsAndProbes()
+
 	if err := obs.InitSensorManager(); err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	var cancelWg sync.WaitGroup
-
 	/* Remove any stale programs, otherwise feature set change can cause
 	 * old programs to linger resulting in undefined behavior. And because
 	 * we recapture current running state from proc and/or have cache of
@@ -209,6 +228,9 @@ func tetragonExecute() error {
 		return err
 	}
 
+	var cancelWg sync.WaitGroup
+	defer cancelWg.Wait()
+
 	pm, err := tetragonGrpc.NewProcessManager(
 		ctx,
 		&cancelWg,
@@ -226,16 +248,6 @@ func tetragonExecute() error {
 		}
 	}
 
-	go func() {
-		<-sigs
-		obs.PrintStats()
-		obs.RemovePrograms()
-		stopProfile()
-		cancel()
-		cancelWg.Wait()
-		os.Exit(1)
-	}()
-
 	log.WithField("enabled", exportFilename != "").WithField("fileName", exportFilename).Info("Exporter configuration")
 	obs.AddListener(pm)
 	saveInitInfo()

pkg/observer/observer.go

@@ -209,7 +209,7 @@ func (k *Observer) runEvents(stopCtx context.Context, ready func()) error {
 		for stopCtx.Err() == nil {
 			record, err := perfReader.Read()
 			if err != nil {
-				// NOTE(JM): Keeping the old behaviour for now and just counting the errors without stopping
+				// NOTE(JM and Djalal): count and log errors while excluding the stopping context
 				if stopCtx.Err() == nil {
 					k.errorCntr++
 					ringbufmetrics.ErrorsSet(float64(k.errorCntr))