added interactive shell recovery in case of disconnect, enhanced alerting to reset automatically on recovery
This commit is contained in:
parent
2343f8e970
commit
a9fd652eb7
|
@ -20,7 +20,6 @@ const (
|
|||
type BarChart struct {
|
||||
*ui.Block
|
||||
*data.Consumer
|
||||
alert *data.Alert
|
||||
bars []Bar
|
||||
scale int
|
||||
maxValue float64
|
||||
|
@ -56,7 +55,7 @@ func NewBarChart(c config.BarChartConfig, palette console.Palette) *BarChart {
|
|||
case sample := <-chart.SampleChannel:
|
||||
chart.consumeSample(sample)
|
||||
case alert := <-chart.AlertChannel:
|
||||
chart.alert = alert
|
||||
chart.Alert = alert
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
@ -70,12 +69,10 @@ func (b *BarChart) consumeSample(sample *data.Sample) {
|
|||
|
||||
float, err := util.ParseFloat(sample.Value)
|
||||
if err != nil {
|
||||
b.AlertChannel <- &data.Alert{
|
||||
Title: "FAILED TO PARSE A NUMBER",
|
||||
Text: err.Error(),
|
||||
Color: sample.Color,
|
||||
}
|
||||
b.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||
return
|
||||
} else {
|
||||
b.HandleConsumeSuccess()
|
||||
}
|
||||
|
||||
index := -1
|
||||
|
@ -163,5 +160,5 @@ func (b *BarChart) Draw(buffer *ui.Buffer) {
|
|||
barXCoordinate += barWidth + barIndent
|
||||
}
|
||||
|
||||
component.RenderAlert(b.alert, b.Rectangle, buffer)
|
||||
component.RenderAlert(b.Alert, b.Rectangle, buffer)
|
||||
}
|
||||
|
|
|
@ -20,7 +20,6 @@ const (
|
|||
type Gauge struct {
|
||||
*ui.Block
|
||||
*data.Consumer
|
||||
alert *data.Alert
|
||||
minValue float64
|
||||
maxValue float64
|
||||
curValue float64
|
||||
|
@ -31,7 +30,7 @@ type Gauge struct {
|
|||
|
||||
func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
|
||||
|
||||
gauge := Gauge{
|
||||
g := Gauge{
|
||||
Block: component.NewBlock(c.Title, true, palette),
|
||||
Consumer: data.NewConsumer(),
|
||||
scale: *c.Scale,
|
||||
|
@ -42,27 +41,25 @@ func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
|
|||
go func() {
|
||||
for {
|
||||
select {
|
||||
case sample := <-gauge.SampleChannel:
|
||||
gauge.ConsumeSample(sample)
|
||||
case alert := <-gauge.AlertChannel:
|
||||
gauge.alert = alert
|
||||
case sample := <-g.SampleChannel:
|
||||
g.ConsumeSample(sample)
|
||||
case alert := <-g.AlertChannel:
|
||||
g.Alert = alert
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return &gauge
|
||||
return &g
|
||||
}
|
||||
|
||||
func (g *Gauge) ConsumeSample(sample *data.Sample) {
|
||||
|
||||
float, err := util.ParseFloat(sample.Value)
|
||||
if err != nil {
|
||||
g.AlertChannel <- &data.Alert{
|
||||
Title: "FAILED TO PARSE A NUMBER",
|
||||
Text: err.Error(),
|
||||
Color: sample.Color,
|
||||
}
|
||||
g.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||
return
|
||||
} else {
|
||||
g.HandleConsumeSuccess()
|
||||
}
|
||||
|
||||
switch sample.Label {
|
||||
|
@ -114,5 +111,5 @@ func (g *Gauge) Draw(buffer *ui.Buffer) {
|
|||
}
|
||||
}
|
||||
|
||||
component.RenderAlert(g.alert, g.Rectangle, buffer)
|
||||
component.RenderAlert(g.Alert, g.Rectangle, buffer)
|
||||
}
|
||||
|
|
|
@ -41,7 +41,6 @@ const (
|
|||
type RunChart struct {
|
||||
*ui.Block
|
||||
*data.Consumer
|
||||
alert *data.Alert
|
||||
lines []TimeLine
|
||||
grid ChartGrid
|
||||
timescale time.Duration
|
||||
|
@ -102,7 +101,7 @@ func NewRunChart(c config.RunChartConfig, palette console.Palette) *RunChart {
|
|||
case sample := <-chart.SampleChannel:
|
||||
chart.consumeSample(sample)
|
||||
case alert := <-chart.AlertChannel:
|
||||
chart.alert = alert
|
||||
chart.Alert = alert
|
||||
case command := <-chart.CommandChannel:
|
||||
switch command.Type {
|
||||
case CommandDisableSelection:
|
||||
|
@ -140,7 +139,7 @@ func (c *RunChart) Draw(buffer *ui.Buffer) {
|
|||
c.renderAxes(buffer)
|
||||
c.renderLines(buffer, drawArea)
|
||||
c.renderLegend(buffer, drawArea)
|
||||
component.RenderAlert(c.alert, c.Rectangle, buffer)
|
||||
component.RenderAlert(c.Alert, c.Rectangle, buffer)
|
||||
c.mutex.Unlock()
|
||||
}
|
||||
|
||||
|
@ -158,12 +157,10 @@ func (c *RunChart) consumeSample(sample *data.Sample) {
|
|||
|
||||
float, err := util.ParseFloat(sample.Value)
|
||||
if err != nil {
|
||||
c.AlertChannel <- &data.Alert{
|
||||
Title: "FAILED TO PARSE A NUMBER",
|
||||
Text: err.Error(),
|
||||
Color: sample.Color,
|
||||
}
|
||||
c.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||
return
|
||||
} else {
|
||||
c.HandleConsumeSuccess()
|
||||
}
|
||||
|
||||
c.mutex.Lock()
|
||||
|
|
|
@ -14,7 +14,6 @@ import (
|
|||
type SparkLine struct {
|
||||
*ui.Block
|
||||
*data.Consumer
|
||||
alert *data.Alert
|
||||
values []float64
|
||||
maxValue float64
|
||||
minValue float64
|
||||
|
@ -42,7 +41,7 @@ func NewSparkLine(c config.SparkLineConfig, palette console.Palette) *SparkLine
|
|||
case sample := <-line.SampleChannel:
|
||||
line.consumeSample(sample)
|
||||
case alert := <-line.AlertChannel:
|
||||
line.alert = alert
|
||||
line.Alert = alert
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
@ -54,12 +53,10 @@ func (s *SparkLine) consumeSample(sample *data.Sample) {
|
|||
|
||||
float, err := util.ParseFloat(sample.Value)
|
||||
if err != nil {
|
||||
s.AlertChannel <- &data.Alert{
|
||||
Title: "FAILED TO PARSE A NUMBER",
|
||||
Text: err.Error(),
|
||||
Color: sample.Color,
|
||||
}
|
||||
s.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||
return
|
||||
} else {
|
||||
s.HandleConsumeSuccess()
|
||||
}
|
||||
|
||||
s.values = append(s.values, float)
|
||||
|
@ -141,5 +138,5 @@ func (s *SparkLine) Draw(buffer *ui.Buffer) {
|
|||
s.mutex.Unlock()
|
||||
|
||||
s.Block.Draw(buffer)
|
||||
component.RenderAlert(s.alert, s.Rectangle, buffer)
|
||||
component.RenderAlert(s.Alert, s.Rectangle, buffer)
|
||||
}
|
||||
|
|
|
@ -1,11 +1,30 @@
|
|||
package data
|
||||
|
||||
import ui "github.com/gizak/termui/v3"
|
||||
import (
|
||||
ui "github.com/gizak/termui/v3"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Consumer struct {
|
||||
SampleChannel chan *Sample
|
||||
AlertChannel chan *Alert
|
||||
CommandChannel chan *Command
|
||||
Alert *Alert
|
||||
}
|
||||
|
||||
func (c *Consumer) HandleConsumeSuccess() {
|
||||
if c.Alert != nil && c.Alert.Recoverable {
|
||||
c.Alert = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Consumer) HandleConsumeFailure(title string, err error, sample *Sample) {
|
||||
c.AlertChannel <- &Alert{
|
||||
Title: strings.ToUpper(title),
|
||||
Text: err.Error(),
|
||||
Color: sample.Color,
|
||||
Recoverable: true,
|
||||
}
|
||||
}
|
||||
|
||||
type Sample struct {
|
||||
|
@ -15,9 +34,10 @@ type Sample struct {
|
|||
}
|
||||
|
||||
type Alert struct {
|
||||
Title string
|
||||
Text string
|
||||
Color *ui.Color
|
||||
Title string
|
||||
Text string
|
||||
Color *ui.Color
|
||||
Recoverable bool
|
||||
}
|
||||
|
||||
type Command struct {
|
||||
|
|
|
@ -72,7 +72,7 @@ func (s *BasicInteractiveShell) execute() (string, error) {
|
|||
if err != nil {
|
||||
s.errCount++
|
||||
if s.errCount > errorThreshold {
|
||||
s.item.ptyShell = nil // restart session
|
||||
s.item.basicShell = nil // restart session
|
||||
}
|
||||
return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err))
|
||||
}
|
||||
|
|
|
@ -58,9 +58,10 @@ func (s *Sampler) sample(item *Item, options config.Options) {
|
|||
s.triggersChannel <- sample
|
||||
} else if err != nil {
|
||||
s.consumer.AlertChannel <- &Alert{
|
||||
Title: "SAMPLING FAILURE",
|
||||
Text: getErrorMessage(err),
|
||||
Color: item.color,
|
||||
Title: "Sampling failure",
|
||||
Text: getErrorMessage(err),
|
||||
Color: item.color,
|
||||
Recoverable: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -80,9 +80,10 @@ func (t *Trigger) Execute(sample *Sample) {
|
|||
|
||||
if t.actions.visual {
|
||||
t.consumer.AlertChannel <- &Alert{
|
||||
Title: t.title,
|
||||
Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value),
|
||||
Color: sample.Color,
|
||||
Title: t.title,
|
||||
Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value),
|
||||
Color: sample.Color,
|
||||
Recoverable: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -106,9 +107,10 @@ func (t *Trigger) evaluate(sample *Sample) bool {
|
|||
|
||||
if err != nil {
|
||||
t.consumer.AlertChannel <- &Alert{
|
||||
Title: "TRIGGER CONDITION FAILURE",
|
||||
Text: getErrorMessage(err),
|
||||
Color: sample.Color,
|
||||
Title: "Trigger condition failure",
|
||||
Text: getErrorMessage(err),
|
||||
Color: sample.Color,
|
||||
Recoverable: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -45,7 +45,8 @@ barcharts:
|
|||
init: $mongoconnection
|
||||
sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount()
|
||||
- label: IN_PROCESS
|
||||
sample: echo 0
|
||||
init: $mongoconnection
|
||||
sample: db.getCollection('posts').find({status:'UNKNOWN'}).itcount()
|
||||
- label: FAILED
|
||||
init: $mongoconnection
|
||||
sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()
|
||||
|
|
Loading…
Reference in New Issue