added interactive shell recovery in case of disconnect, enhanced alerting to reset automatically on recovery

component/barchart/barchart.go

@@ -20,7 +20,6 @@ const (
 type BarChart struct {
 	*ui.Block
 	*data.Consumer
-	alert    *data.Alert
 	bars     []Bar
 	scale    int
 	maxValue float64
@@ -56,7 +55,7 @@ func NewBarChart(c config.BarChartConfig, palette console.Palette) *BarChart {
 			case sample := <-chart.SampleChannel:
 				chart.consumeSample(sample)
 			case alert := <-chart.AlertChannel:
-				chart.alert = alert
+				chart.Alert = alert
 			}
 		}
 	}()
@@ -70,12 +69,10 @@ func (b *BarChart) consumeSample(sample *data.Sample) {
 
 	float, err := util.ParseFloat(sample.Value)
 	if err != nil {
-		b.AlertChannel <- &data.Alert{
+		b.HandleConsumeFailure("Failed to parse a number", err, sample)
-			Title: "FAILED TO PARSE A NUMBER",
-			Text:  err.Error(),
-			Color: sample.Color,
-		}
 		return
+	} else {
+		b.HandleConsumeSuccess()
 	}
 
 	index := -1
@@ -163,5 +160,5 @@ func (b *BarChart) Draw(buffer *ui.Buffer) {
 		barXCoordinate += barWidth + barIndent
 	}
 
-	component.RenderAlert(b.alert, b.Rectangle, buffer)
+	component.RenderAlert(b.Alert, b.Rectangle, buffer)
 }

component/gauge/gauge.go

@@ -20,7 +20,6 @@ const (
 type Gauge struct {
 	*ui.Block
 	*data.Consumer
-	alert    *data.Alert
 	minValue float64
 	maxValue float64
 	curValue float64
@@ -31,7 +30,7 @@ type Gauge struct {
 
 func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
 
-	gauge := Gauge{
+	g := Gauge{
 		Block:    component.NewBlock(c.Title, true, palette),
 		Consumer: data.NewConsumer(),
 		scale:    *c.Scale,
@@ -42,27 +41,25 @@ func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
 	go func() {
 		for {
 			select {
-			case sample := <-gauge.SampleChannel:
+			case sample := <-g.SampleChannel:
-				gauge.ConsumeSample(sample)
+				g.ConsumeSample(sample)
-			case alert := <-gauge.AlertChannel:
+			case alert := <-g.AlertChannel:
-				gauge.alert = alert
+				g.Alert = alert
 			}
 		}
 	}()
 
-	return &gauge
+	return &g
 }
 
 func (g *Gauge) ConsumeSample(sample *data.Sample) {
 
 	float, err := util.ParseFloat(sample.Value)
 	if err != nil {
-		g.AlertChannel <- &data.Alert{
+		g.HandleConsumeFailure("Failed to parse a number", err, sample)
-			Title: "FAILED TO PARSE A NUMBER",
-			Text:  err.Error(),
-			Color: sample.Color,
-		}
 		return
+	} else {
+		g.HandleConsumeSuccess()
 	}
 
 	switch sample.Label {
@@ -114,5 +111,5 @@ func (g *Gauge) Draw(buffer *ui.Buffer) {
 		}
 	}
 
-	component.RenderAlert(g.alert, g.Rectangle, buffer)
+	component.RenderAlert(g.Alert, g.Rectangle, buffer)
 }

component/runchart/runchart.go

@@ -41,7 +41,6 @@ const (
 type RunChart struct {
 	*ui.Block
 	*data.Consumer
-	alert     *data.Alert
 	lines     []TimeLine
 	grid      ChartGrid
 	timescale time.Duration
@@ -102,7 +101,7 @@ func NewRunChart(c config.RunChartConfig, palette console.Palette) *RunChart {
 			case sample := <-chart.SampleChannel:
 				chart.consumeSample(sample)
 			case alert := <-chart.AlertChannel:
-				chart.alert = alert
+				chart.Alert = alert
 			case command := <-chart.CommandChannel:
 				switch command.Type {
 				case CommandDisableSelection:
@@ -140,7 +139,7 @@ func (c *RunChart) Draw(buffer *ui.Buffer) {
 	c.renderAxes(buffer)
 	c.renderLines(buffer, drawArea)
 	c.renderLegend(buffer, drawArea)
-	component.RenderAlert(c.alert, c.Rectangle, buffer)
+	component.RenderAlert(c.Alert, c.Rectangle, buffer)
 	c.mutex.Unlock()
 }
 
@@ -158,12 +157,10 @@ func (c *RunChart) consumeSample(sample *data.Sample) {
 
 	float, err := util.ParseFloat(sample.Value)
 	if err != nil {
-		c.AlertChannel <- &data.Alert{
+		c.HandleConsumeFailure("Failed to parse a number", err, sample)
-			Title: "FAILED TO PARSE A NUMBER",
-			Text:  err.Error(),
-			Color: sample.Color,
-		}
 		return
+	} else {
+		c.HandleConsumeSuccess()
 	}
 
 	c.mutex.Lock()

component/sparkline/sparkline.go

@@ -14,7 +14,6 @@ import (
 type SparkLine struct {
 	*ui.Block
 	*data.Consumer
-	alert    *data.Alert
 	values   []float64
 	maxValue float64
 	minValue float64
@@ -42,7 +41,7 @@ func NewSparkLine(c config.SparkLineConfig, palette console.Palette) *SparkLine
 			case sample := <-line.SampleChannel:
 				line.consumeSample(sample)
 			case alert := <-line.AlertChannel:
-				line.alert = alert
+				line.Alert = alert
 			}
 		}
 	}()
@@ -54,12 +53,10 @@ func (s *SparkLine) consumeSample(sample *data.Sample) {
 
 	float, err := util.ParseFloat(sample.Value)
 	if err != nil {
-		s.AlertChannel <- &data.Alert{
+		s.HandleConsumeFailure("Failed to parse a number", err, sample)
-			Title: "FAILED TO PARSE A NUMBER",
-			Text:  err.Error(),
-			Color: sample.Color,
-		}
 		return
+	} else {
+		s.HandleConsumeSuccess()
 	}
 
 	s.values = append(s.values, float)
@@ -141,5 +138,5 @@ func (s *SparkLine) Draw(buffer *ui.Buffer) {
 	s.mutex.Unlock()
 
 	s.Block.Draw(buffer)
-	component.RenderAlert(s.alert, s.Rectangle, buffer)
+	component.RenderAlert(s.Alert, s.Rectangle, buffer)
 }

data/consumer.go

@@ -1,11 +1,30 @@
 package data
 
-import ui "github.com/gizak/termui/v3"
+import (
+	ui "github.com/gizak/termui/v3"
+	"strings"
+)
 
 type Consumer struct {
 	SampleChannel  chan *Sample
 	AlertChannel   chan *Alert
 	CommandChannel chan *Command
+	Alert          *Alert
+}
+
+func (c *Consumer) HandleConsumeSuccess() {
+	if c.Alert != nil && c.Alert.Recoverable {
+		c.Alert = nil
+	}
+}
+
+func (c *Consumer) HandleConsumeFailure(title string, err error, sample *Sample) {
+	c.AlertChannel <- &Alert{
+		Title:       strings.ToUpper(title),
+		Text:        err.Error(),
+		Color:       sample.Color,
+		Recoverable: true,
+	}
 }
 
 type Sample struct {
@@ -18,6 +37,7 @@ type Alert struct {
 	Title       string
 	Text        string
 	Color       *ui.Color
+	Recoverable bool
 }
 
 type Command struct {

data/int_shell.go

@@ -72,7 +72,7 @@ func (s *BasicInteractiveShell) execute() (string, error) {
 	if err != nil {
 		s.errCount++
 		if s.errCount > errorThreshold {
-			s.item.ptyShell = nil // restart session
+			s.item.basicShell = nil // restart session
 		}
 		return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err))
 	}

data/sampler.go

@@ -58,9 +58,10 @@ func (s *Sampler) sample(item *Item, options config.Options) {
 		s.triggersChannel <- sample
 	} else if err != nil {
 		s.consumer.AlertChannel <- &Alert{
-			Title: "SAMPLING FAILURE",
+			Title:       "Sampling failure",
 			Text:        getErrorMessage(err),
 			Color:       item.color,
+			Recoverable: true,
 		}
 	}
 }

data/trigger.go

@@ -83,6 +83,7 @@ func (t *Trigger) Execute(sample *Sample) {
 				Title:       t.title,
 				Text:        fmt.Sprintf("%s: %v", sample.Label, sample.Value),
 				Color:       sample.Color,
+				Recoverable: false,
 			}
 		}
 
@@ -106,9 +107,10 @@ func (t *Trigger) evaluate(sample *Sample) bool {
 
 	if err != nil {
 		t.consumer.AlertChannel <- &Alert{
-			Title: "TRIGGER CONDITION FAILURE",
+			Title:       "Trigger condition failure",
 			Text:        getErrorMessage(err),
 			Color:       sample.Color,
+			Recoverable: true,
 		}
 	}
 

example.yml

@@ -45,7 +45,8 @@ barcharts:
     init: $mongoconnection
     sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount()
   - label: IN_PROCESS
-    sample: echo 0
+    init: $mongoconnection
+    sample: db.getCollection('posts').find({status:'UNKNOWN'}).itcount()
   - label: FAILED
     init: $mongoconnection
     sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()