tetragon: fix graceful shutdown and exit code

cmd/tetragon/main.go

@@ -119,6 +119,9 @@ func stopProfile() {
 }
 
 func tetragonExecute() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
@@ -143,6 +146,12 @@ func tetragonExecute() error {
 		defaults.NetnsDir = viper.GetString(keyNetnsDir)
 	}
 
+	// Setup file system mounts
+	bpf.CheckOrMountFS("")
+	bpf.CheckOrMountDebugFS()
+	bpf.CheckOrMountCgroup2()
+
+	// Start profilers first as we have to capture them in signal handling
 	if memProfile != "" {
 		log.WithField("file", memProfile).Info("Starting mem profiling")
 	}
@@ -160,25 +169,35 @@ func tetragonExecute() error {
 		log.WithField("file", cpuProfile).Info("Starting cpu profiling")
 	}
 
-	bpf.CheckOrMountFS("")
-	bpf.CheckOrMountDebugFS()
-	bpf.CheckOrMountCgroup2()
-
-	sensors.LogRegisteredSensorsAndProbes()
+	defer stopProfile()
 
+	// Raise memory resource
 	bpf.ConfigureResourceLimits()
+
+	// Get observer bpf maps and programs directory
 	observerDir := getObserverDir()
 	option.Config.BpfDir = observerDir
 	option.Config.MapDir = observerDir
+
+	// Get observer from configFile
 	obs := observer.NewObserver(configFile)
+	defer func() {
+		obs.PrintStats()
+		obs.RemovePrograms()
+	}()
+
+	go func() {
+		s := <-sigs
+		log.Infof("Received signal %s, shutting down...", s)
+		cancel()
+	}()
+
+	sensors.LogRegisteredSensorsAndProbes()
+
 	if err := obs.InitSensorManager(); err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	var cancelWg sync.WaitGroup
-
 	/* Remove any stale programs, otherwise feature set change can cause
 	 * old programs to linger resulting in undefined behavior. And because
 	 * we recapture current running state from proc and/or have cache of
@@ -209,6 +228,9 @@ func tetragonExecute() error {
 		return err
 	}
 
+	var cancelWg sync.WaitGroup
+	defer cancelWg.Wait()
+
 	pm, err := tetragonGrpc.NewProcessManager(
 		ctx,
 		&cancelWg,
@@ -226,16 +248,6 @@ func tetragonExecute() error {
 		}
 	}
 
-	go func() {
-		<-sigs
-		obs.PrintStats()
-		obs.RemovePrograms()
-		stopProfile()
-		cancel()
-		cancelWg.Wait()
-		os.Exit(1)
-	}()
-
 	log.WithField("enabled", exportFilename != "").WithField("fileName", exportFilename).Info("Exporter configuration")
 	obs.AddListener(pm)
 	saveInitInfo()

pkg/observer/observer.go

@@ -209,7 +209,7 @@ func (k *Observer) runEvents(stopCtx context.Context, ready func()) error {
 		for stopCtx.Err() == nil {
 			record, err := perfReader.Read()
 			if err != nil {
-				// NOTE(JM): Keeping the old behaviour for now and just counting the errors without stopping
+				// NOTE(JM and Djalal): count and log errors while excluding the stopping context
 				if stopCtx.Err() == nil {
 					k.errorCntr++
 					ringbufmetrics.ErrorsSet(float64(k.errorCntr))