From a9fd652eb7db37bb29c3170fda030bcb604c6689 Mon Sep 17 00:00:00 2001 From: sqshq Date: Wed, 5 Jun 2019 23:07:34 -0400 Subject: [PATCH] added interactive shell recovery in case of disconnect, enhanced alerting to reset automatically on recovery --- component/barchart/barchart.go | 13 +++++-------- component/gauge/gauge.go | 23 ++++++++++------------- component/runchart/runchart.go | 13 +++++-------- component/sparkline/sparkline.go | 13 +++++-------- data/consumer.go | 28 ++++++++++++++++++++++++---- data/int_shell.go | 2 +- data/sampler.go | 7 ++++--- data/trigger.go | 14 ++++++++------ example.yml | 3 ++- 9 files changed, 64 insertions(+), 52 deletions(-) diff --git a/component/barchart/barchart.go b/component/barchart/barchart.go index 5803ae6..2295709 100644 --- a/component/barchart/barchart.go +++ b/component/barchart/barchart.go @@ -20,7 +20,6 @@ const ( type BarChart struct { *ui.Block *data.Consumer - alert *data.Alert bars []Bar scale int maxValue float64 @@ -56,7 +55,7 @@ func NewBarChart(c config.BarChartConfig, palette console.Palette) *BarChart { case sample := <-chart.SampleChannel: chart.consumeSample(sample) case alert := <-chart.AlertChannel: - chart.alert = alert + chart.Alert = alert } } }() @@ -70,12 +69,10 @@ func (b *BarChart) consumeSample(sample *data.Sample) { float, err := util.ParseFloat(sample.Value) if err != nil { - b.AlertChannel <- &data.Alert{ - Title: "FAILED TO PARSE A NUMBER", - Text: err.Error(), - Color: sample.Color, - } + b.HandleConsumeFailure("Failed to parse a number", err, sample) return + } else { + b.HandleConsumeSuccess() } index := -1 @@ -163,5 +160,5 @@ func (b *BarChart) Draw(buffer *ui.Buffer) { barXCoordinate += barWidth + barIndent } - component.RenderAlert(b.alert, b.Rectangle, buffer) + component.RenderAlert(b.Alert, b.Rectangle, buffer) } diff --git a/component/gauge/gauge.go b/component/gauge/gauge.go index afddf28..9192822 100644 --- a/component/gauge/gauge.go +++ b/component/gauge/gauge.go @@ -20,7 +20,6 @@ const ( type Gauge struct { *ui.Block *data.Consumer - alert *data.Alert minValue float64 maxValue float64 curValue float64 @@ -31,7 +30,7 @@ type Gauge struct { func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge { - gauge := Gauge{ + g := Gauge{ Block: component.NewBlock(c.Title, true, palette), Consumer: data.NewConsumer(), scale: *c.Scale, @@ -42,27 +41,25 @@ func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge { go func() { for { select { - case sample := <-gauge.SampleChannel: - gauge.ConsumeSample(sample) - case alert := <-gauge.AlertChannel: - gauge.alert = alert + case sample := <-g.SampleChannel: + g.ConsumeSample(sample) + case alert := <-g.AlertChannel: + g.Alert = alert } } }() - return &gauge + return &g } func (g *Gauge) ConsumeSample(sample *data.Sample) { float, err := util.ParseFloat(sample.Value) if err != nil { - g.AlertChannel <- &data.Alert{ - Title: "FAILED TO PARSE A NUMBER", - Text: err.Error(), - Color: sample.Color, - } + g.HandleConsumeFailure("Failed to parse a number", err, sample) return + } else { + g.HandleConsumeSuccess() } switch sample.Label { @@ -114,5 +111,5 @@ func (g *Gauge) Draw(buffer *ui.Buffer) { } } - component.RenderAlert(g.alert, g.Rectangle, buffer) + component.RenderAlert(g.Alert, g.Rectangle, buffer) } diff --git a/component/runchart/runchart.go b/component/runchart/runchart.go index bc4a232..2ec7449 100644 --- a/component/runchart/runchart.go +++ b/component/runchart/runchart.go @@ -41,7 +41,6 @@ const ( type RunChart struct { *ui.Block *data.Consumer - alert *data.Alert lines []TimeLine grid ChartGrid timescale time.Duration @@ -102,7 +101,7 @@ func NewRunChart(c config.RunChartConfig, palette console.Palette) *RunChart { case sample := <-chart.SampleChannel: chart.consumeSample(sample) case alert := <-chart.AlertChannel: - chart.alert = alert + chart.Alert = alert case command := <-chart.CommandChannel: switch command.Type { case CommandDisableSelection: @@ -140,7 +139,7 @@ func (c *RunChart) Draw(buffer *ui.Buffer) { c.renderAxes(buffer) c.renderLines(buffer, drawArea) c.renderLegend(buffer, drawArea) - component.RenderAlert(c.alert, c.Rectangle, buffer) + component.RenderAlert(c.Alert, c.Rectangle, buffer) c.mutex.Unlock() } @@ -158,12 +157,10 @@ func (c *RunChart) consumeSample(sample *data.Sample) { float, err := util.ParseFloat(sample.Value) if err != nil { - c.AlertChannel <- &data.Alert{ - Title: "FAILED TO PARSE A NUMBER", - Text: err.Error(), - Color: sample.Color, - } + c.HandleConsumeFailure("Failed to parse a number", err, sample) return + } else { + c.HandleConsumeSuccess() } c.mutex.Lock() diff --git a/component/sparkline/sparkline.go b/component/sparkline/sparkline.go index 3bb4184..f65f5a2 100644 --- a/component/sparkline/sparkline.go +++ b/component/sparkline/sparkline.go @@ -14,7 +14,6 @@ import ( type SparkLine struct { *ui.Block *data.Consumer - alert *data.Alert values []float64 maxValue float64 minValue float64 @@ -42,7 +41,7 @@ func NewSparkLine(c config.SparkLineConfig, palette console.Palette) *SparkLine case sample := <-line.SampleChannel: line.consumeSample(sample) case alert := <-line.AlertChannel: - line.alert = alert + line.Alert = alert } } }() @@ -54,12 +53,10 @@ func (s *SparkLine) consumeSample(sample *data.Sample) { float, err := util.ParseFloat(sample.Value) if err != nil { - s.AlertChannel <- &data.Alert{ - Title: "FAILED TO PARSE A NUMBER", - Text: err.Error(), - Color: sample.Color, - } + s.HandleConsumeFailure("Failed to parse a number", err, sample) return + } else { + s.HandleConsumeSuccess() } s.values = append(s.values, float) @@ -141,5 +138,5 @@ func (s *SparkLine) Draw(buffer *ui.Buffer) { s.mutex.Unlock() s.Block.Draw(buffer) - component.RenderAlert(s.alert, s.Rectangle, buffer) + component.RenderAlert(s.Alert, s.Rectangle, buffer) } diff --git a/data/consumer.go b/data/consumer.go index a410b99..5031f17 100644 --- a/data/consumer.go +++ b/data/consumer.go @@ -1,11 +1,30 @@ package data -import ui "github.com/gizak/termui/v3" +import ( + ui "github.com/gizak/termui/v3" + "strings" +) type Consumer struct { SampleChannel chan *Sample AlertChannel chan *Alert CommandChannel chan *Command + Alert *Alert +} + +func (c *Consumer) HandleConsumeSuccess() { + if c.Alert != nil && c.Alert.Recoverable { + c.Alert = nil + } +} + +func (c *Consumer) HandleConsumeFailure(title string, err error, sample *Sample) { + c.AlertChannel <- &Alert{ + Title: strings.ToUpper(title), + Text: err.Error(), + Color: sample.Color, + Recoverable: true, + } } type Sample struct { @@ -15,9 +34,10 @@ type Sample struct { } type Alert struct { - Title string - Text string - Color *ui.Color + Title string + Text string + Color *ui.Color + Recoverable bool } type Command struct { diff --git a/data/int_shell.go b/data/int_shell.go index 1bfd8f5..f6b5c3a 100644 --- a/data/int_shell.go +++ b/data/int_shell.go @@ -72,7 +72,7 @@ func (s *BasicInteractiveShell) execute() (string, error) { if err != nil { s.errCount++ if s.errCount > errorThreshold { - s.item.ptyShell = nil // restart session + s.item.basicShell = nil // restart session } return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err)) } diff --git a/data/sampler.go b/data/sampler.go index f0ec171..6bdff16 100644 --- a/data/sampler.go +++ b/data/sampler.go @@ -58,9 +58,10 @@ func (s *Sampler) sample(item *Item, options config.Options) { s.triggersChannel <- sample } else if err != nil { s.consumer.AlertChannel <- &Alert{ - Title: "SAMPLING FAILURE", - Text: getErrorMessage(err), - Color: item.color, + Title: "Sampling failure", + Text: getErrorMessage(err), + Color: item.color, + Recoverable: true, } } } diff --git a/data/trigger.go b/data/trigger.go index ebc449f..16c7480 100644 --- a/data/trigger.go +++ b/data/trigger.go @@ -80,9 +80,10 @@ func (t *Trigger) Execute(sample *Sample) { if t.actions.visual { t.consumer.AlertChannel <- &Alert{ - Title: t.title, - Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value), - Color: sample.Color, + Title: t.title, + Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value), + Color: sample.Color, + Recoverable: false, } } @@ -106,9 +107,10 @@ func (t *Trigger) evaluate(sample *Sample) bool { if err != nil { t.consumer.AlertChannel <- &Alert{ - Title: "TRIGGER CONDITION FAILURE", - Text: getErrorMessage(err), - Color: sample.Color, + Title: "Trigger condition failure", + Text: getErrorMessage(err), + Color: sample.Color, + Recoverable: true, } } diff --git a/example.yml b/example.yml index 82a00ef..3220a5d 100644 --- a/example.yml +++ b/example.yml @@ -45,7 +45,8 @@ barcharts: init: $mongoconnection sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount() - label: IN_PROCESS - sample: echo 0 + init: $mongoconnection + sample: db.getCollection('posts').find({status:'UNKNOWN'}).itcount() - label: FAILED init: $mongoconnection sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()