Commit 36073169 by Carl Bergquist Committed by GitHub

Merge pull request #6489 from bergquist/reduce_notification_states

Reduce alerting states
parents 56340bfd b2d8e83c
...@@ -40,8 +40,7 @@ var ( ...@@ -40,8 +40,7 @@ var (
M_Alerting_Result_State_Ok Counter M_Alerting_Result_State_Ok Counter
M_Alerting_Result_State_Paused Counter M_Alerting_Result_State_Paused Counter
M_Alerting_Result_State_NoData Counter M_Alerting_Result_State_NoData Counter
M_Alerting_Result_State_ExecError Counter M_Alerting_Result_State_Pending Counter
M_Alerting_Result_State_Pending Counter
M_Alerting_Active_Alerts Counter M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter M_Alerting_Notification_Sent_Slack Counter
M_Alerting_Notification_Sent_Email Counter M_Alerting_Notification_Sent_Email Counter
...@@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) { ...@@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) {
M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok") M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused") M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data") M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error")
M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending") M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts") M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
......
...@@ -9,35 +9,47 @@ import ( ...@@ -9,35 +9,47 @@ import (
type AlertStateType string type AlertStateType string
type AlertSeverityType string type AlertSeverityType string
type NoDataOption string type NoDataOption string
type ExecutionErrorOption string
const ( const (
AlertStateNoData AlertStateType = "no_data" AlertStateNoData AlertStateType = "no_data"
AlertStateExecError AlertStateType = "execution_error" AlertStatePaused AlertStateType = "paused"
AlertStatePaused AlertStateType = "paused" AlertStateAlerting AlertStateType = "alerting"
AlertStateAlerting AlertStateType = "alerting" AlertStateOK AlertStateType = "ok"
AlertStateOK AlertStateType = "ok" AlertStatePending AlertStateType = "pending"
AlertStatePending AlertStateType = "pending"
) )
const ( const (
NoDataSetNoData NoDataOption = "no_data" NoDataSetNoData NoDataOption = "no_data"
NoDataSetAlerting NoDataOption = "alerting" NoDataSetAlerting NoDataOption = "alerting"
NoDataSetOK NoDataOption = "ok"
NoDataKeepState NoDataOption = "keep_state" NoDataKeepState NoDataOption = "keep_state"
) )
const (
ExecutionErrorSetAlerting ExecutionErrorOption = "alerting"
ExecutionErrorKeepState ExecutionErrorOption = "keep_state"
)
func (s AlertStateType) IsValid() bool { func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStatePending return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
} }
func (s NoDataOption) IsValid() bool { func (s NoDataOption) IsValid() bool {
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataSetOK || s == NoDataKeepState return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataKeepState
} }
func (s NoDataOption) ToAlertState() AlertStateType { func (s NoDataOption) ToAlertState() AlertStateType {
return AlertStateType(s) return AlertStateType(s)
} }
func (s ExecutionErrorOption) IsValid() bool {
return s == ExecutionErrorSetAlerting || s == ExecutionErrorKeepState
}
func (s ExecutionErrorOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}
type Alert struct { type Alert struct {
Id int64 Id int64
Version int64 Version int64
......
...@@ -26,10 +26,23 @@ type EvalContext struct { ...@@ -26,10 +26,23 @@ type EvalContext struct {
ImagePublicUrl string ImagePublicUrl string
ImageOnDiskPath string ImageOnDiskPath string
NoDataFound bool NoDataFound bool
PrevAlertState m.AlertStateType
Ctx context.Context Ctx context.Context
} }
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
PrevAlertState: rule.State,
}
}
type StateDescription struct { type StateDescription struct {
Color string Color string
Text string Text string
...@@ -48,11 +61,6 @@ func (c *EvalContext) GetStateModel() *StateDescription { ...@@ -48,11 +61,6 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#888888", Color: "#888888",
Text: "No Data", Text: "No Data",
} }
case m.AlertStateExecError:
return &StateDescription{
Color: "#000",
Text: "Execution Error",
}
case m.AlertStateAlerting: case m.AlertStateAlerting:
return &StateDescription{ return &StateDescription{
Color: "#D63232", Color: "#D63232",
...@@ -63,6 +71,18 @@ func (c *EvalContext) GetStateModel() *StateDescription { ...@@ -63,6 +71,18 @@ func (c *EvalContext) GetStateModel() *StateDescription {
} }
} }
func (c *EvalContext) ShouldUpdateAlertState() bool {
return c.Rule.State != c.PrevAlertState
}
func (c *EvalContext) ShouldSendNotification() bool {
if (c.PrevAlertState == m.AlertStatePending) && (c.Rule.State == m.AlertStateOK) {
return false
}
return true
}
func (a *EvalContext) GetDurationMs() float64 { func (a *EvalContext) GetDurationMs() float64 {
return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000) return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
} }
...@@ -97,14 +117,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) { ...@@ -97,14 +117,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return ruleUrl, nil return ruleUrl, nil
} }
} }
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
package alerting
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertingEvalContext(t *testing.T) {
Convey("Eval context", t, func() {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
Convey("Should update alert state", func() {
Convey("ok -> alerting", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateAlerting
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> ok", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateOK
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
})
Convey("Should send notifications", func() {
Convey("pending -> ok", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.State = models.AlertStateOK
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> alerting", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateAlerting
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
})
})
}
...@@ -27,32 +27,55 @@ func NewResultHandler() *DefaultResultHandler { ...@@ -27,32 +27,55 @@ func NewResultHandler() *DefaultResultHandler {
} }
} }
func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { func (handler *DefaultResultHandler) GetStateFromEvaluation(evalContext *EvalContext) m.AlertStateType {
oldState := evalContext.Rule.State if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"error", evalContext.Error,
"changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState())
if evalContext.Rule.ExecutionErrorState == m.ExecutionErrorKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.ExecutionErrorState.ToAlertState()
}
} else if evalContext.Firing {
return m.AlertStateAlerting
} else if evalContext.NoDataFound {
handler.log.Info("Alert Rule returned no data",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"changing state to", evalContext.Rule.NoDataState.ToAlertState())
if evalContext.Rule.NoDataState == m.NoDataKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.NoDataState.ToAlertState()
}
}
return m.AlertStateOK
}
func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
executionError := "" executionError := ""
annotationData := simplejson.New() annotationData := simplejson.New()
evalContext.Rule.State = handler.GetStateFromEvaluation(evalContext)
if evalContext.Error != nil { if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", evalContext.Rule.Id, "error", evalContext.Error)
evalContext.Rule.State = m.AlertStateExecError
executionError = evalContext.Error.Error() executionError = evalContext.Error.Error()
annotationData.Set("errorMessage", executionError) annotationData.Set("errorMessage", executionError)
} else if evalContext.Firing { }
evalContext.Rule.State = m.AlertStateAlerting
if evalContext.Firing {
annotationData = simplejson.NewFromAny(evalContext.EvalMatches) annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
} else {
if evalContext.NoDataFound {
if evalContext.Rule.NoDataState != m.NoDataKeepState {
evalContext.Rule.State = evalContext.Rule.NoDataState.ToAlertState()
}
} else {
evalContext.Rule.State = m.AlertStateOK
}
} }
countStateResult(evalContext.Rule.State) countStateResult(evalContext.Rule.State)
if handler.shouldUpdateAlertState(evalContext, oldState) { if evalContext.ShouldUpdateAlertState() {
handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "oldState", oldState) handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "prev state", evalContext.PrevAlertState)
cmd := &m.SetAlertStateCommand{ cmd := &m.SetAlertStateCommand{
AlertId: evalContext.Rule.Id, AlertId: evalContext.Rule.Id,
...@@ -76,7 +99,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { ...@@ -76,7 +99,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
Title: evalContext.Rule.Name, Title: evalContext.Rule.Name,
Text: evalContext.GetStateModel().Text, Text: evalContext.GetStateModel().Text,
NewState: string(evalContext.Rule.State), NewState: string(evalContext.Rule.State),
PrevState: string(oldState), PrevState: string(evalContext.PrevAlertState),
Epoch: time.Now().Unix(), Epoch: time.Now().Unix(),
Data: annotationData, Data: annotationData,
} }
...@@ -86,21 +109,14 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { ...@@ -86,21 +109,14 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
handler.log.Error("Failed to save annotation for new alert state", "error", err) handler.log.Error("Failed to save annotation for new alert state", "error", err)
} }
if (oldState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) { if evalContext.ShouldSendNotification() {
handler.log.Info("Notfication not sent", "oldState", oldState, "newState", evalContext.Rule.State)
} else {
handler.notifier.Notify(evalContext) handler.notifier.Notify(evalContext)
} }
} }
return nil return nil
} }
func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalContext, oldState m.AlertStateType) bool {
return evalContext.Rule.State != oldState
}
func countStateResult(state m.AlertStateType) { func countStateResult(state m.AlertStateType) {
switch state { switch state {
case m.AlertStatePending: case m.AlertStatePending:
...@@ -113,7 +129,5 @@ func countStateResult(state m.AlertStateType) { ...@@ -113,7 +129,5 @@ func countStateResult(state m.AlertStateType) {
metrics.M_Alerting_Result_State_Paused.Inc(1) metrics.M_Alerting_Result_State_Paused.Inc(1)
case m.AlertStateNoData: case m.AlertStateNoData:
metrics.M_Alerting_Result_State_NoData.Inc(1) metrics.M_Alerting_Result_State_NoData.Inc(1)
case m.AlertStateExecError:
metrics.M_Alerting_Result_State_ExecError.Inc(1)
} }
} }
package alerting package alerting
// import ( import (
// "context" "context"
// "testing" "testing"
//
// "github.com/grafana/grafana/pkg/models" "fmt"
// . "github.com/smartystreets/goconvey/convey"
// ) "github.com/grafana/grafana/pkg/models"
// . "github.com/smartystreets/goconvey/convey"
// func TestAlertResultHandler(t *testing.T) { )
// Convey("Test result Handler", t, func() {
// func TestAlertingResultHandler(t *testing.T) {
// handler := NewResultHandler() Convey("Result handler", t, func() {
// evalContext := NewEvalContext(context.TODO(), &Rule{}) ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
// dummieError := fmt.Errorf("dummie")
// Convey("Should update", func() { handler := NewResultHandler()
//
// Convey("when no earlier alert state", func() { Convey("Should update alert state", func() {
// oldState := models.AlertStateOK
// Convey("ok -> alerting", func() {
// evalContext.Rule.State = models.AlertStateAlerting ctx.PrevAlertState = models.AlertStateOK
// evalContext.Rule.NoDataState = models.NoDataKeepState ctx.Firing = true
// evalContext.NoDataFound = true
// So(handler.GetStateFromEvaluation(ctx), ShouldEqual, models.AlertStateAlerting)
// So(handler.shouldUpdateAlertState(evalContext, oldState), ShouldBeFalse) So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
// }) })
// })
// }) Convey("ok -> error(alerting)", func() {
// } ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
})
})
}
...@@ -11,17 +11,18 @@ import ( ...@@ -11,17 +11,18 @@ import (
) )
type Rule struct { type Rule struct {
Id int64 Id int64
OrgId int64 OrgId int64
DashboardId int64 DashboardId int64
PanelId int64 PanelId int64
Frequency int64 Frequency int64
Name string Name string
Message string Message string
NoDataState m.NoDataOption NoDataState m.NoDataOption
State m.AlertStateType ExecutionErrorState m.ExecutionErrorOption
Conditions []Condition State m.AlertStateType
Notifications []int64 Conditions []Condition
Notifications []int64
} }
type ValidationError struct { type ValidationError struct {
...@@ -77,6 +78,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { ...@@ -77,6 +78,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Frequency = ruleDef.Frequency model.Frequency = ruleDef.Frequency
model.State = ruleDef.State model.State = ruleDef.State
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
for _, v := range ruleDef.Settings.Get("notifications").MustArray() { for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v) jsonModel := simplejson.NewFromAny(v)
......
...@@ -37,10 +37,14 @@ var reducerTypes = [ ...@@ -37,10 +37,14 @@ var reducerTypes = [
]; ];
var noDataModes = [ var noDataModes = [
{text: 'OK', value: 'ok'},
{text: 'Alerting', value: 'alerting'}, {text: 'Alerting', value: 'alerting'},
{text: 'No Data', value: 'no_data'}, {text: 'No Data', value: 'no_data'},
{text: 'Keep Last', value: 'keep_last'}, {text: 'Keep Last State', value: 'keep_state'},
];
var executionErrorModes = [
{text: 'Alerting', value: 'alerting'},
{text: 'Keep Last State', value: 'keep_state'},
]; ];
function createReducerPart(model) { function createReducerPart(model) {
...@@ -48,7 +52,6 @@ function createReducerPart(model) { ...@@ -48,7 +52,6 @@ function createReducerPart(model) {
return new QueryPart(model, def); return new QueryPart(model, def);
} }
function getStateDisplayModel(state) { function getStateDisplayModel(state) {
switch (state) { switch (state) {
case 'ok': { case 'ok': {
...@@ -113,6 +116,7 @@ export default { ...@@ -113,6 +116,7 @@ export default {
conditionTypes: conditionTypes, conditionTypes: conditionTypes,
evalFunctions: evalFunctions, evalFunctions: evalFunctions,
noDataModes: noDataModes, noDataModes: noDataModes,
executionErrorModes: executionErrorModes,
reducerTypes: reducerTypes, reducerTypes: reducerTypes,
createReducerPart: createReducerPart, createReducerPart: createReducerPart,
joinEvalMatches: joinEvalMatches, joinEvalMatches: joinEvalMatches,
......
...@@ -19,6 +19,7 @@ export class AlertTabCtrl { ...@@ -19,6 +19,7 @@ export class AlertTabCtrl {
conditionModels: any; conditionModels: any;
evalFunctions: any; evalFunctions: any;
noDataModes: any; noDataModes: any;
executionErrorModes: any;
addNotificationSegment; addNotificationSegment;
notifications; notifications;
alertNotifications; alertNotifications;
...@@ -42,6 +43,7 @@ export class AlertTabCtrl { ...@@ -42,6 +43,7 @@ export class AlertTabCtrl {
this.evalFunctions = alertDef.evalFunctions; this.evalFunctions = alertDef.evalFunctions;
this.conditionTypes = alertDef.conditionTypes; this.conditionTypes = alertDef.conditionTypes;
this.noDataModes = alertDef.noDataModes; this.noDataModes = alertDef.noDataModes;
this.executionErrorModes = alertDef.executionErrorModes;
this.appSubUrl = config.appSubUrl; this.appSubUrl = config.appSubUrl;
} }
...@@ -140,6 +142,7 @@ export class AlertTabCtrl { ...@@ -140,6 +142,7 @@ export class AlertTabCtrl {
} }
alert.noDataState = alert.noDataState || 'no_data'; alert.noDataState = alert.noDataState || 'no_data';
alert.executionErrorState = alert.executionErrorState || 'alerting';
alert.frequency = alert.frequency || '60s'; alert.frequency = alert.frequency || '60s';
alert.handler = alert.handler || 1; alert.handler = alert.handler || 1;
alert.notifications = alert.notifications || []; alert.notifications = alert.notifications || [];
......
...@@ -82,7 +82,7 @@ ...@@ -82,7 +82,7 @@
<div class="gf-form-group"> <div class="gf-form-group">
<div class="gf-form"> <div class="gf-form">
<span class="gf-form-label">If no data points or all values are null</span> <span class="gf-form-label width-18">If no data points or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span> <span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper"> <div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes"> <select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
...@@ -90,6 +90,15 @@ ...@@ -90,6 +90,15 @@
</div> </div>
</div> </div>
<div class="gf-form">
<span class="gf-form-label width-18">On execution error or timeout</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form-button-row"> <div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()"> <button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule Test Rule
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment