added interactive shell recovery in case of disconnect, enhanced alerting to reset automatically on recovery

This commit is contained in:
sqshq 2019-06-05 23:07:34 -04:00
parent 2343f8e970
commit a9fd652eb7
9 changed files with 64 additions and 52 deletions

View File

@ -20,7 +20,6 @@ const (
type BarChart struct { type BarChart struct {
*ui.Block *ui.Block
*data.Consumer *data.Consumer
alert *data.Alert
bars []Bar bars []Bar
scale int scale int
maxValue float64 maxValue float64
@ -56,7 +55,7 @@ func NewBarChart(c config.BarChartConfig, palette console.Palette) *BarChart {
case sample := <-chart.SampleChannel: case sample := <-chart.SampleChannel:
chart.consumeSample(sample) chart.consumeSample(sample)
case alert := <-chart.AlertChannel: case alert := <-chart.AlertChannel:
chart.alert = alert chart.Alert = alert
} }
} }
}() }()
@ -70,12 +69,10 @@ func (b *BarChart) consumeSample(sample *data.Sample) {
float, err := util.ParseFloat(sample.Value) float, err := util.ParseFloat(sample.Value)
if err != nil { if err != nil {
b.AlertChannel <- &data.Alert{ b.HandleConsumeFailure("Failed to parse a number", err, sample)
Title: "FAILED TO PARSE A NUMBER",
Text: err.Error(),
Color: sample.Color,
}
return return
} else {
b.HandleConsumeSuccess()
} }
index := -1 index := -1
@ -163,5 +160,5 @@ func (b *BarChart) Draw(buffer *ui.Buffer) {
barXCoordinate += barWidth + barIndent barXCoordinate += barWidth + barIndent
} }
component.RenderAlert(b.alert, b.Rectangle, buffer) component.RenderAlert(b.Alert, b.Rectangle, buffer)
} }

View File

@ -20,7 +20,6 @@ const (
type Gauge struct { type Gauge struct {
*ui.Block *ui.Block
*data.Consumer *data.Consumer
alert *data.Alert
minValue float64 minValue float64
maxValue float64 maxValue float64
curValue float64 curValue float64
@ -31,7 +30,7 @@ type Gauge struct {
func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge { func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
gauge := Gauge{ g := Gauge{
Block: component.NewBlock(c.Title, true, palette), Block: component.NewBlock(c.Title, true, palette),
Consumer: data.NewConsumer(), Consumer: data.NewConsumer(),
scale: *c.Scale, scale: *c.Scale,
@ -42,27 +41,25 @@ func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
go func() { go func() {
for { for {
select { select {
case sample := <-gauge.SampleChannel: case sample := <-g.SampleChannel:
gauge.ConsumeSample(sample) g.ConsumeSample(sample)
case alert := <-gauge.AlertChannel: case alert := <-g.AlertChannel:
gauge.alert = alert g.Alert = alert
} }
} }
}() }()
return &gauge return &g
} }
func (g *Gauge) ConsumeSample(sample *data.Sample) { func (g *Gauge) ConsumeSample(sample *data.Sample) {
float, err := util.ParseFloat(sample.Value) float, err := util.ParseFloat(sample.Value)
if err != nil { if err != nil {
g.AlertChannel <- &data.Alert{ g.HandleConsumeFailure("Failed to parse a number", err, sample)
Title: "FAILED TO PARSE A NUMBER",
Text: err.Error(),
Color: sample.Color,
}
return return
} else {
g.HandleConsumeSuccess()
} }
switch sample.Label { switch sample.Label {
@ -114,5 +111,5 @@ func (g *Gauge) Draw(buffer *ui.Buffer) {
} }
} }
component.RenderAlert(g.alert, g.Rectangle, buffer) component.RenderAlert(g.Alert, g.Rectangle, buffer)
} }

View File

@ -41,7 +41,6 @@ const (
type RunChart struct { type RunChart struct {
*ui.Block *ui.Block
*data.Consumer *data.Consumer
alert *data.Alert
lines []TimeLine lines []TimeLine
grid ChartGrid grid ChartGrid
timescale time.Duration timescale time.Duration
@ -102,7 +101,7 @@ func NewRunChart(c config.RunChartConfig, palette console.Palette) *RunChart {
case sample := <-chart.SampleChannel: case sample := <-chart.SampleChannel:
chart.consumeSample(sample) chart.consumeSample(sample)
case alert := <-chart.AlertChannel: case alert := <-chart.AlertChannel:
chart.alert = alert chart.Alert = alert
case command := <-chart.CommandChannel: case command := <-chart.CommandChannel:
switch command.Type { switch command.Type {
case CommandDisableSelection: case CommandDisableSelection:
@ -140,7 +139,7 @@ func (c *RunChart) Draw(buffer *ui.Buffer) {
c.renderAxes(buffer) c.renderAxes(buffer)
c.renderLines(buffer, drawArea) c.renderLines(buffer, drawArea)
c.renderLegend(buffer, drawArea) c.renderLegend(buffer, drawArea)
component.RenderAlert(c.alert, c.Rectangle, buffer) component.RenderAlert(c.Alert, c.Rectangle, buffer)
c.mutex.Unlock() c.mutex.Unlock()
} }
@ -158,12 +157,10 @@ func (c *RunChart) consumeSample(sample *data.Sample) {
float, err := util.ParseFloat(sample.Value) float, err := util.ParseFloat(sample.Value)
if err != nil { if err != nil {
c.AlertChannel <- &data.Alert{ c.HandleConsumeFailure("Failed to parse a number", err, sample)
Title: "FAILED TO PARSE A NUMBER",
Text: err.Error(),
Color: sample.Color,
}
return return
} else {
c.HandleConsumeSuccess()
} }
c.mutex.Lock() c.mutex.Lock()

View File

@ -14,7 +14,6 @@ import (
type SparkLine struct { type SparkLine struct {
*ui.Block *ui.Block
*data.Consumer *data.Consumer
alert *data.Alert
values []float64 values []float64
maxValue float64 maxValue float64
minValue float64 minValue float64
@ -42,7 +41,7 @@ func NewSparkLine(c config.SparkLineConfig, palette console.Palette) *SparkLine
case sample := <-line.SampleChannel: case sample := <-line.SampleChannel:
line.consumeSample(sample) line.consumeSample(sample)
case alert := <-line.AlertChannel: case alert := <-line.AlertChannel:
line.alert = alert line.Alert = alert
} }
} }
}() }()
@ -54,12 +53,10 @@ func (s *SparkLine) consumeSample(sample *data.Sample) {
float, err := util.ParseFloat(sample.Value) float, err := util.ParseFloat(sample.Value)
if err != nil { if err != nil {
s.AlertChannel <- &data.Alert{ s.HandleConsumeFailure("Failed to parse a number", err, sample)
Title: "FAILED TO PARSE A NUMBER",
Text: err.Error(),
Color: sample.Color,
}
return return
} else {
s.HandleConsumeSuccess()
} }
s.values = append(s.values, float) s.values = append(s.values, float)
@ -141,5 +138,5 @@ func (s *SparkLine) Draw(buffer *ui.Buffer) {
s.mutex.Unlock() s.mutex.Unlock()
s.Block.Draw(buffer) s.Block.Draw(buffer)
component.RenderAlert(s.alert, s.Rectangle, buffer) component.RenderAlert(s.Alert, s.Rectangle, buffer)
} }

View File

@ -1,11 +1,30 @@
package data package data
import ui "github.com/gizak/termui/v3" import (
ui "github.com/gizak/termui/v3"
"strings"
)
type Consumer struct { type Consumer struct {
SampleChannel chan *Sample SampleChannel chan *Sample
AlertChannel chan *Alert AlertChannel chan *Alert
CommandChannel chan *Command CommandChannel chan *Command
Alert *Alert
}
func (c *Consumer) HandleConsumeSuccess() {
if c.Alert != nil && c.Alert.Recoverable {
c.Alert = nil
}
}
func (c *Consumer) HandleConsumeFailure(title string, err error, sample *Sample) {
c.AlertChannel <- &Alert{
Title: strings.ToUpper(title),
Text: err.Error(),
Color: sample.Color,
Recoverable: true,
}
} }
type Sample struct { type Sample struct {
@ -18,6 +37,7 @@ type Alert struct {
Title string Title string
Text string Text string
Color *ui.Color Color *ui.Color
Recoverable bool
} }
type Command struct { type Command struct {

View File

@ -72,7 +72,7 @@ func (s *BasicInteractiveShell) execute() (string, error) {
if err != nil { if err != nil {
s.errCount++ s.errCount++
if s.errCount > errorThreshold { if s.errCount > errorThreshold {
s.item.ptyShell = nil // restart session s.item.basicShell = nil // restart session
} }
return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err)) return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err))
} }

View File

@ -58,9 +58,10 @@ func (s *Sampler) sample(item *Item, options config.Options) {
s.triggersChannel <- sample s.triggersChannel <- sample
} else if err != nil { } else if err != nil {
s.consumer.AlertChannel <- &Alert{ s.consumer.AlertChannel <- &Alert{
Title: "SAMPLING FAILURE", Title: "Sampling failure",
Text: getErrorMessage(err), Text: getErrorMessage(err),
Color: item.color, Color: item.color,
Recoverable: true,
} }
} }
} }

View File

@ -83,6 +83,7 @@ func (t *Trigger) Execute(sample *Sample) {
Title: t.title, Title: t.title,
Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value), Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value),
Color: sample.Color, Color: sample.Color,
Recoverable: false,
} }
} }
@ -106,9 +107,10 @@ func (t *Trigger) evaluate(sample *Sample) bool {
if err != nil { if err != nil {
t.consumer.AlertChannel <- &Alert{ t.consumer.AlertChannel <- &Alert{
Title: "TRIGGER CONDITION FAILURE", Title: "Trigger condition failure",
Text: getErrorMessage(err), Text: getErrorMessage(err),
Color: sample.Color, Color: sample.Color,
Recoverable: true,
} }
} }

View File

@ -45,7 +45,8 @@ barcharts:
init: $mongoconnection init: $mongoconnection
sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount() sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount()
- label: IN_PROCESS - label: IN_PROCESS
sample: echo 0 init: $mongoconnection
sample: db.getCollection('posts').find({status:'UNKNOWN'}).itcount()
- label: FAILED - label: FAILED
init: $mongoconnection init: $mongoconnection
sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount() sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()