added interactive shell recovery in case of disconnect, enhanced alerting to reset automatically on recovery
This commit is contained in:
parent
2343f8e970
commit
a9fd652eb7
|
@ -20,7 +20,6 @@ const (
|
||||||
type BarChart struct {
|
type BarChart struct {
|
||||||
*ui.Block
|
*ui.Block
|
||||||
*data.Consumer
|
*data.Consumer
|
||||||
alert *data.Alert
|
|
||||||
bars []Bar
|
bars []Bar
|
||||||
scale int
|
scale int
|
||||||
maxValue float64
|
maxValue float64
|
||||||
|
@ -56,7 +55,7 @@ func NewBarChart(c config.BarChartConfig, palette console.Palette) *BarChart {
|
||||||
case sample := <-chart.SampleChannel:
|
case sample := <-chart.SampleChannel:
|
||||||
chart.consumeSample(sample)
|
chart.consumeSample(sample)
|
||||||
case alert := <-chart.AlertChannel:
|
case alert := <-chart.AlertChannel:
|
||||||
chart.alert = alert
|
chart.Alert = alert
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
@ -70,12 +69,10 @@ func (b *BarChart) consumeSample(sample *data.Sample) {
|
||||||
|
|
||||||
float, err := util.ParseFloat(sample.Value)
|
float, err := util.ParseFloat(sample.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.AlertChannel <- &data.Alert{
|
b.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||||
Title: "FAILED TO PARSE A NUMBER",
|
|
||||||
Text: err.Error(),
|
|
||||||
Color: sample.Color,
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
|
} else {
|
||||||
|
b.HandleConsumeSuccess()
|
||||||
}
|
}
|
||||||
|
|
||||||
index := -1
|
index := -1
|
||||||
|
@ -163,5 +160,5 @@ func (b *BarChart) Draw(buffer *ui.Buffer) {
|
||||||
barXCoordinate += barWidth + barIndent
|
barXCoordinate += barWidth + barIndent
|
||||||
}
|
}
|
||||||
|
|
||||||
component.RenderAlert(b.alert, b.Rectangle, buffer)
|
component.RenderAlert(b.Alert, b.Rectangle, buffer)
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,7 +20,6 @@ const (
|
||||||
type Gauge struct {
|
type Gauge struct {
|
||||||
*ui.Block
|
*ui.Block
|
||||||
*data.Consumer
|
*data.Consumer
|
||||||
alert *data.Alert
|
|
||||||
minValue float64
|
minValue float64
|
||||||
maxValue float64
|
maxValue float64
|
||||||
curValue float64
|
curValue float64
|
||||||
|
@ -31,7 +30,7 @@ type Gauge struct {
|
||||||
|
|
||||||
func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
|
func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
|
||||||
|
|
||||||
gauge := Gauge{
|
g := Gauge{
|
||||||
Block: component.NewBlock(c.Title, true, palette),
|
Block: component.NewBlock(c.Title, true, palette),
|
||||||
Consumer: data.NewConsumer(),
|
Consumer: data.NewConsumer(),
|
||||||
scale: *c.Scale,
|
scale: *c.Scale,
|
||||||
|
@ -42,27 +41,25 @@ func NewGauge(c config.GaugeConfig, palette console.Palette) *Gauge {
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case sample := <-gauge.SampleChannel:
|
case sample := <-g.SampleChannel:
|
||||||
gauge.ConsumeSample(sample)
|
g.ConsumeSample(sample)
|
||||||
case alert := <-gauge.AlertChannel:
|
case alert := <-g.AlertChannel:
|
||||||
gauge.alert = alert
|
g.Alert = alert
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return &gauge
|
return &g
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Gauge) ConsumeSample(sample *data.Sample) {
|
func (g *Gauge) ConsumeSample(sample *data.Sample) {
|
||||||
|
|
||||||
float, err := util.ParseFloat(sample.Value)
|
float, err := util.ParseFloat(sample.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
g.AlertChannel <- &data.Alert{
|
g.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||||
Title: "FAILED TO PARSE A NUMBER",
|
|
||||||
Text: err.Error(),
|
|
||||||
Color: sample.Color,
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
|
} else {
|
||||||
|
g.HandleConsumeSuccess()
|
||||||
}
|
}
|
||||||
|
|
||||||
switch sample.Label {
|
switch sample.Label {
|
||||||
|
@ -114,5 +111,5 @@ func (g *Gauge) Draw(buffer *ui.Buffer) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
component.RenderAlert(g.alert, g.Rectangle, buffer)
|
component.RenderAlert(g.Alert, g.Rectangle, buffer)
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,6 @@ const (
|
||||||
type RunChart struct {
|
type RunChart struct {
|
||||||
*ui.Block
|
*ui.Block
|
||||||
*data.Consumer
|
*data.Consumer
|
||||||
alert *data.Alert
|
|
||||||
lines []TimeLine
|
lines []TimeLine
|
||||||
grid ChartGrid
|
grid ChartGrid
|
||||||
timescale time.Duration
|
timescale time.Duration
|
||||||
|
@ -102,7 +101,7 @@ func NewRunChart(c config.RunChartConfig, palette console.Palette) *RunChart {
|
||||||
case sample := <-chart.SampleChannel:
|
case sample := <-chart.SampleChannel:
|
||||||
chart.consumeSample(sample)
|
chart.consumeSample(sample)
|
||||||
case alert := <-chart.AlertChannel:
|
case alert := <-chart.AlertChannel:
|
||||||
chart.alert = alert
|
chart.Alert = alert
|
||||||
case command := <-chart.CommandChannel:
|
case command := <-chart.CommandChannel:
|
||||||
switch command.Type {
|
switch command.Type {
|
||||||
case CommandDisableSelection:
|
case CommandDisableSelection:
|
||||||
|
@ -140,7 +139,7 @@ func (c *RunChart) Draw(buffer *ui.Buffer) {
|
||||||
c.renderAxes(buffer)
|
c.renderAxes(buffer)
|
||||||
c.renderLines(buffer, drawArea)
|
c.renderLines(buffer, drawArea)
|
||||||
c.renderLegend(buffer, drawArea)
|
c.renderLegend(buffer, drawArea)
|
||||||
component.RenderAlert(c.alert, c.Rectangle, buffer)
|
component.RenderAlert(c.Alert, c.Rectangle, buffer)
|
||||||
c.mutex.Unlock()
|
c.mutex.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -158,12 +157,10 @@ func (c *RunChart) consumeSample(sample *data.Sample) {
|
||||||
|
|
||||||
float, err := util.ParseFloat(sample.Value)
|
float, err := util.ParseFloat(sample.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.AlertChannel <- &data.Alert{
|
c.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||||
Title: "FAILED TO PARSE A NUMBER",
|
|
||||||
Text: err.Error(),
|
|
||||||
Color: sample.Color,
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
|
} else {
|
||||||
|
c.HandleConsumeSuccess()
|
||||||
}
|
}
|
||||||
|
|
||||||
c.mutex.Lock()
|
c.mutex.Lock()
|
||||||
|
|
|
@ -14,7 +14,6 @@ import (
|
||||||
type SparkLine struct {
|
type SparkLine struct {
|
||||||
*ui.Block
|
*ui.Block
|
||||||
*data.Consumer
|
*data.Consumer
|
||||||
alert *data.Alert
|
|
||||||
values []float64
|
values []float64
|
||||||
maxValue float64
|
maxValue float64
|
||||||
minValue float64
|
minValue float64
|
||||||
|
@ -42,7 +41,7 @@ func NewSparkLine(c config.SparkLineConfig, palette console.Palette) *SparkLine
|
||||||
case sample := <-line.SampleChannel:
|
case sample := <-line.SampleChannel:
|
||||||
line.consumeSample(sample)
|
line.consumeSample(sample)
|
||||||
case alert := <-line.AlertChannel:
|
case alert := <-line.AlertChannel:
|
||||||
line.alert = alert
|
line.Alert = alert
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
@ -54,12 +53,10 @@ func (s *SparkLine) consumeSample(sample *data.Sample) {
|
||||||
|
|
||||||
float, err := util.ParseFloat(sample.Value)
|
float, err := util.ParseFloat(sample.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.AlertChannel <- &data.Alert{
|
s.HandleConsumeFailure("Failed to parse a number", err, sample)
|
||||||
Title: "FAILED TO PARSE A NUMBER",
|
|
||||||
Text: err.Error(),
|
|
||||||
Color: sample.Color,
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
|
} else {
|
||||||
|
s.HandleConsumeSuccess()
|
||||||
}
|
}
|
||||||
|
|
||||||
s.values = append(s.values, float)
|
s.values = append(s.values, float)
|
||||||
|
@ -141,5 +138,5 @@ func (s *SparkLine) Draw(buffer *ui.Buffer) {
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
|
|
||||||
s.Block.Draw(buffer)
|
s.Block.Draw(buffer)
|
||||||
component.RenderAlert(s.alert, s.Rectangle, buffer)
|
component.RenderAlert(s.Alert, s.Rectangle, buffer)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,30 @@
|
||||||
package data
|
package data
|
||||||
|
|
||||||
import ui "github.com/gizak/termui/v3"
|
import (
|
||||||
|
ui "github.com/gizak/termui/v3"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
type Consumer struct {
|
type Consumer struct {
|
||||||
SampleChannel chan *Sample
|
SampleChannel chan *Sample
|
||||||
AlertChannel chan *Alert
|
AlertChannel chan *Alert
|
||||||
CommandChannel chan *Command
|
CommandChannel chan *Command
|
||||||
|
Alert *Alert
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Consumer) HandleConsumeSuccess() {
|
||||||
|
if c.Alert != nil && c.Alert.Recoverable {
|
||||||
|
c.Alert = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Consumer) HandleConsumeFailure(title string, err error, sample *Sample) {
|
||||||
|
c.AlertChannel <- &Alert{
|
||||||
|
Title: strings.ToUpper(title),
|
||||||
|
Text: err.Error(),
|
||||||
|
Color: sample.Color,
|
||||||
|
Recoverable: true,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Sample struct {
|
type Sample struct {
|
||||||
|
@ -18,6 +37,7 @@ type Alert struct {
|
||||||
Title string
|
Title string
|
||||||
Text string
|
Text string
|
||||||
Color *ui.Color
|
Color *ui.Color
|
||||||
|
Recoverable bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type Command struct {
|
type Command struct {
|
||||||
|
|
|
@ -72,7 +72,7 @@ func (s *BasicInteractiveShell) execute() (string, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.errCount++
|
s.errCount++
|
||||||
if s.errCount > errorThreshold {
|
if s.errCount > errorThreshold {
|
||||||
s.item.ptyShell = nil // restart session
|
s.item.basicShell = nil // restart session
|
||||||
}
|
}
|
||||||
return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err))
|
return "", errors.New(fmt.Sprintf("Failed to execute command: %s", err))
|
||||||
}
|
}
|
||||||
|
|
|
@ -58,9 +58,10 @@ func (s *Sampler) sample(item *Item, options config.Options) {
|
||||||
s.triggersChannel <- sample
|
s.triggersChannel <- sample
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
s.consumer.AlertChannel <- &Alert{
|
s.consumer.AlertChannel <- &Alert{
|
||||||
Title: "SAMPLING FAILURE",
|
Title: "Sampling failure",
|
||||||
Text: getErrorMessage(err),
|
Text: getErrorMessage(err),
|
||||||
Color: item.color,
|
Color: item.color,
|
||||||
|
Recoverable: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -83,6 +83,7 @@ func (t *Trigger) Execute(sample *Sample) {
|
||||||
Title: t.title,
|
Title: t.title,
|
||||||
Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value),
|
Text: fmt.Sprintf("%s: %v", sample.Label, sample.Value),
|
||||||
Color: sample.Color,
|
Color: sample.Color,
|
||||||
|
Recoverable: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,9 +107,10 @@ func (t *Trigger) evaluate(sample *Sample) bool {
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.consumer.AlertChannel <- &Alert{
|
t.consumer.AlertChannel <- &Alert{
|
||||||
Title: "TRIGGER CONDITION FAILURE",
|
Title: "Trigger condition failure",
|
||||||
Text: getErrorMessage(err),
|
Text: getErrorMessage(err),
|
||||||
Color: sample.Color,
|
Color: sample.Color,
|
||||||
|
Recoverable: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,7 +45,8 @@ barcharts:
|
||||||
init: $mongoconnection
|
init: $mongoconnection
|
||||||
sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount()
|
sample: db.getCollection('posts').find({status:'INACTIVE'}).itcount()
|
||||||
- label: IN_PROCESS
|
- label: IN_PROCESS
|
||||||
sample: echo 0
|
init: $mongoconnection
|
||||||
|
sample: db.getCollection('posts').find({status:'UNKNOWN'}).itcount()
|
||||||
- label: FAILED
|
- label: FAILED
|
||||||
init: $mongoconnection
|
init: $mongoconnection
|
||||||
sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()
|
sample: db.getCollection('posts').find({status:'ACTIVE'}).itcount()
|
||||||
|
|
Loading…
Reference in New Issue