package fm import ( "database/sql" "encoding/json" "fmt" "io" "net/http" "strconv" "strings" "time" "ems.agt/lib/dborm" "ems.agt/lib/global" "ems.agt/lib/log" "ems.agt/lib/services" "ems.agt/restagent/config" "xorm.io/xorm" "github.com/go-resty/resty/v2" _ "github.com/go-sql-driver/mysql" "github.com/gorilla/mux" ) const ( AlarmStatusClear = 0 AlarmStatusActive = 1 ) const ( ClearTypeUnclear = 0 ClearTypeAutoClear = 1 ClearTypeManualClear = 2 ) const ( AckStateUnacked = 0 AckStateAcked = 1 ) const ( AlarmTypeCommunicationAlarm = 1 AlarmTypeEquipmentAlarm = 2 AlarmTypeProcessingFailure = 3 AlarmTypeEnvironmentalAlarm = 4 AlarmTypeQualityOfServiceAlarm = 5 ) const ( AlarmPerceivedSeverityCritical = 1 AlarmPerceivedSeverityMajor = 2 AlarmPerceivedSeverityMinor = 3 AlarmPerceivedSeverityWarning = 4 AlarmPerceivedSeverityEvent = 5 ) const ( AlarmSeqBeginNumber = 1 ) type Response struct { Data interface{} `json:"data"` } type Alarm struct { AlarmSeq int `json:"alarmSeq"` AlarmId string `json:"alarmId" xorm:"alarm_id"` NeId string `json:"neId"` AlarmCode int `json:"alarmCode"` AlarmTitle string `json:"alarmTitle"` EventTime string `json:"eventTime"` AlarmType string `json:"alarmType"` OrigSeverity string `json:"origSeverity"` PerceivedSeverity string `json:"perceivedSeverity"` PVFlag string `json:"pvFlag" xorm:"pv_flag"` NeName string `json:"neName"` NeType string `json:"neType"` ObjectUid string `json:"objectUid" xorm:"object_uid"` ObjectName string `json:"objectName" xorm:"object_name"` ObjectType string `json:"objectType" xorm:"object_type"` LocationInfo string `json:"locationInfo"` Province string `json:"province"` AlarmStatus int `json:"alarmStatus"` SpecificProblem string `json:"specificProblem"` SpecificProblemID string `json:"specificProblemID" xorm:"specific_problem_id"` AddInfo string `json:"addInfo"` AckState int `json:"ackState"` AckTime sql.NullTime `json:"ackTime"` AckUser string `json:"ackUser"` ClearType int `json:"clearType"` // 0: Unclear, 1: Auto clear, 2: Manual clear ClearTime sql.NullTime `json:"clearTime"` } type AlarmLog struct { NeType string `json:"neType" xorm:"ne_type"` NeId string `json:"neId" xorm:"ne_id"` AlarmSeq int `json:"alarmSeq" xorm:"alarm_seq"` AlarmId string `json:"alarmId" xorm:"alarm_id"` AlarmCode int `json:"alarmCode" xorm:"alarm_code"` AlarmStatus int `json:"alarmStatus" xorm:"alarm_status"` EventTime string `json:"eventTime" xorm:"event_time"` // ClearTime sql.NullTime `json:"clearTime" xorm:"clear_time"` LogTime string `json:"logTime" xorm:"-"` } var ( // alarm management UriAlarms = config.UriPrefix + "/faultManagement/{apiVersion}/elementType/{elementTypeValue}/objectType/alarms" UriAlarmsFmt = config.UriPrefix + "/faultManagement/v1/elementType/%s/objectType/alarms" ) var xEngine *xorm.Engine type DatabaseClient struct { dbType string dbUrl string dbConnMaxLifetime time.Duration dbMaxIdleConns int dbMaxOpenConns int IsShowSQL bool XEngine *xorm.Engine } var DbClient DatabaseClient func InitDbClient(dbType, dbUser, dbPassword, dbHost, dbPort, dbName string) error { DbClient.dbUrl = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&parseTime=true&loc=Local", dbUser, dbPassword, dbHost, dbPort, dbName) DbClient.dbType = dbType DbClient.dbConnMaxLifetime = 0 DbClient.dbMaxIdleConns = 0 DbClient.dbMaxOpenConns = 0 if log.GetLevel() == log.LOG_TRACE { DbClient.IsShowSQL = true } log.Debugf("dbType:%s dbUrl:%s:******@tcp(%s:%s)/%s??charset=utf8&parseTime=true&loc=Local", dbType, dbUser, dbHost, dbPort, dbName) var err error DbClient.XEngine, err = xorm.NewEngine(DbClient.dbType, DbClient.dbUrl) if err != nil { log.Error("Failed to connet database:", err) return err } DbClient.XEngine.SetConnMaxLifetime(DbClient.dbConnMaxLifetime) DbClient.XEngine.SetMaxIdleConns(DbClient.dbMaxIdleConns) DbClient.XEngine.SetMaxOpenConns(DbClient.dbMaxOpenConns) if DbClient.IsShowSQL { DbClient.XEngine.ShowSQL(true) } xEngine = DbClient.XEngine return nil } func XormConnectDatabase(dbType, dbUser, dbPassword, dbHost, dbPort, dbName string) (*xorm.Engine, error) { sqlStr := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&parseTime=true&loc=Local", dbUser, dbPassword, dbHost, dbPort, dbName) log.Debugf("dbType:%s Connect to:%s:******@tcp(%s:%s)/%s?charset=utf8&parseTime=true&loc=Local", dbType, dbUser, dbHost, dbPort, dbName) var err error xEngine, err = xorm.NewEngine(dbType, sqlStr) //1、Create xorm engine if err != nil { log.Error("Failed to connect database:", err) return nil, err } if log.GetLevel() == log.LOG_TRACE { xEngine.ShowSQL(true) } return xEngine, nil } func IsNeedToAckAlarm(valueJson *dborm.ValueJson, alarm *Alarm) bool { log.Info("IsNeedToAckAlarm processing... ") if valueJson != nil { status, _ := strconv.Atoi(valueJson.AlarmStatus) log.Tracef("alarm.AlarmStatus=%d, alarm.AlarmType=%s, alarm.OrigSeverity=%s", alarm.AlarmStatus, alarm.AlarmType, alarm.OrigSeverity) log.Tracef("status=%d, valueJson.AlarmType=%s, valueJson.OrigSeverity=%s", status, valueJson.AlarmType, valueJson.OrigSeverity) if alarm.AlarmStatus == status && alarm.AlarmType == valueJson.AlarmType && alarm.OrigSeverity == valueJson.OrigSeverity { return true } } return false } func SetAlarmAckInfo(valueJson *dborm.ValueJson, alarm *Alarm) { log.Info("SetAlarmAckInfo processing... ") alarm.AckState = AckStateAcked alarm.AckTime.Valid = true alarm.AckTime.Time = time.Now() alarm.AckUser = valueJson.AckUser } // process alarm post message from NFs func PostAlarmFromNF(w http.ResponseWriter, r *http.Request) { log.Debug("PostAlarmFromNF processing... ") vars := mux.Vars(r) apiVer := vars["apiVersion"] if apiVer != global.ApiVersionV1 { log.Error("Uri api version is invalid. apiVersion:", apiVer) services.ResponseNotFound404UriNotExist(w, r) return } body, err := io.ReadAll(io.LimitReader(r.Body, global.RequestBodyMaxLen)) if err != nil { log.Error("io.ReadAll is failed:", err) services.ResponseNotFound404UriNotExist(w, r) return } log.Debug("Request body:", string(body)) alarmArray := new([]Alarm) err = json.Unmarshal(body, &alarmArray) if err != nil { log.Error("Failed to Unmarshal:", err) services.ResponseBadRequest400InvalidJson(w) return } valueJson, err := dborm.XormGetAAConfig() if err != nil { log.Error("Failed to XormGetAAConfig:", err) //services.ResponseInternalServerError500ProcessError(w, err) //return } log.Debug("valueJson:", valueJson) session := xEngine.NewSession() defer session.Close() for _, alarmData := range *alarmArray { log.Debug("alarmData:", alarmData) if alarmData.AlarmStatus == AlarmStatusClear { alarmData.ClearType = ClearTypeAutoClear alarmData.ClearTime.Valid = true tm, _ := time.Parse(time.RFC3339, alarmData.EventTime) log.Debugf("EventTime:%s tm:%d tm-datetime:%s", alarmData.EventTime, tm, tm.Local().Format(time.DateTime)) alarmData.ClearTime.Time = tm if IsNeedToAckAlarm(valueJson, &alarmData) == true { SetAlarmAckInfo(valueJson, &alarmData) affected, err := session.Where("ne_type=? and ne_id=? and alarm_id=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.AlarmId). Cols("alarm_status", "clear_type", "clear_time", "ack_state", "ack_time", "ack_user"). Update(alarmData) if err != nil && affected <= 0 { log.Error("Failed to update alarm data:", err) services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } } else { affected, err := session.Where("ne_type=? and ne_id=? and alarm_id=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.AlarmId). Cols("alarm_status", "clear_type", "clear_time"). Update(alarmData) if err != nil && affected <= 0 { log.Error("Failed to update alarm data:", err) services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } } log.Debug("alarmData:", alarmData) var currentSeq string var seq int has, err := xEngine.Table("alarm"). Where("ne_type=? and ne_id=?", alarmData.NeType, alarmData.NeId). Desc("alarm_seq"). Cols("alarm_seq"). Limit(1). Get(¤tSeq) if err != nil { log.Error("Failed to get alarm:", err) continue } if has == true { seq, _ = strconv.Atoi(currentSeq) } eventTime := global.GetFmtTimeString(time.RFC3339, alarmData.EventTime, time.DateTime) alarmLog := new(AlarmLog) alarmLog.NeType = alarmData.NeType alarmLog.NeId = alarmData.NeId alarmLog.AlarmSeq = seq alarmLog.AlarmId = alarmData.AlarmId alarmLog.AlarmCode = alarmData.AlarmCode alarmLog.AlarmStatus = alarmData.AlarmStatus alarmLog.EventTime = eventTime log.Debug("alarmLog:", alarmLog) affected, err := session.Insert(alarmLog) if err != nil && affected <= 0 { log.Error("Failed to insert alarm_log:", err) } // todo: PerceivedSeverity set color var severity string has, err = xEngine.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). //OrderBy("FIELD(orig_severity, 'Critical', 'Major', 'Minor', 'Warning', 'Event') ASC"). Asc("orig_severity"). Cols("orig_severity"). Limit(1). Get(&severity) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, eventTime=%s, severity=%s", alarmData.NeType, alarmData.NeId, alarmData.EventTime, severity) if has == true && severity > alarmData.OrigSeverity { // update exist record _, err := session.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). Update(&Alarm{PerceivedSeverity: severity}) if err != nil { log.Error("Failed to update alarm:", err) continue } } // for alarm forward time format alarmData.EventTime = eventTime } else { has, err := xEngine.Table("alarm"). Where("alarm_id=? and ne_type=? and ne_id=? and alarm_status=1", alarmData.AlarmId, alarmData.NeType, alarmData.NeId). Exist() if err == nil && has == true { log.Warn("Exist the same alarm") continue } var currentSeq string has, err = xEngine.Table("alarm"). Where("ne_type=? and ne_id=?", alarmData.NeType, alarmData.NeId). Desc("alarm_seq"). Cols("alarm_seq"). Limit(1). Get(¤tSeq) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, currentSeq=%s", alarmData.NeType, alarmData.NeId, currentSeq) if has == true { seq, _ := strconv.Atoi(currentSeq) if seq+1 > global.MaxInt32Number { alarmData.AlarmSeq = AlarmSeqBeginNumber } else { alarmData.AlarmSeq = seq + 1 } } else { alarmData.AlarmSeq = AlarmSeqBeginNumber } // todo: PerceivedSeverity set color var severity string has, err = xEngine.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). //OrderBy("FIELD(orig_severity, 'Critical', 'Major', 'Minor', 'Warning', 'Event') ASC"). Asc("orig_severity"). Cols("orig_severity"). Limit(1). Get(&severity) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, eventTime=%s, severity=%s", alarmData.NeType, alarmData.NeId, alarmData.EventTime, severity) if has == false || severity == alarmData.OrigSeverity { alarmData.PerceivedSeverity = alarmData.OrigSeverity } else if severity > alarmData.OrigSeverity { alarmData.PerceivedSeverity = alarmData.OrigSeverity } else { alarmData.PerceivedSeverity = severity // update exist record _, err := session.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). Update(&Alarm{PerceivedSeverity: alarmData.PerceivedSeverity}) if err != nil { log.Error("Failed to update alarm:", err) continue } } eventTime := global.GetFmtTimeString(time.RFC3339, alarmData.EventTime, time.DateTime) alarmData.ObjectUid = alarmData.NeId alarmData.ObjectType = "VNFM" alarmData.EventTime = eventTime if IsNeedToAckAlarm(valueJson, &alarmData) == true { SetAlarmAckInfo(valueJson, &alarmData) } log.Debug("alarmData:", alarmData) affected, err := session.Insert(alarmData) if err != nil && affected <= 0 { log.Error("Failed to insert alarm data:", err) services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } alarmLog := new(AlarmLog) alarmLog.NeType = alarmData.NeType alarmLog.NeId = alarmData.NeId alarmLog.AlarmSeq = alarmData.AlarmSeq alarmLog.AlarmId = alarmData.AlarmId alarmLog.AlarmCode = alarmData.AlarmCode alarmLog.AlarmStatus = alarmData.AlarmStatus alarmLog.EventTime = eventTime log.Debug("alarmLog:", alarmLog) affected, err = session.Insert(alarmLog) if err != nil && affected <= 0 { log.Error("Failed to insert alarm_log:", err) } } if config.GetYamlConfig().Alarm.ForwardAlarm { if err = AlarmEmailForward(&alarmData); err != nil { log.Error("Failed to AlarmEmailForward:", err) } if err = AlarmForwardBySMS(&alarmData); err != nil { log.Error("Failed to AlarmForwardBySMS:", err) } } } session.Commit() services.ResponseStatusOK200Null(w) } // process alarm get from NFs func GetAlarmFromNF(w http.ResponseWriter, r *http.Request) { log.Debug("GetAlarmFromNF processing... ") _, err := services.CheckFrontValidRequest(w, r) if err != nil { log.Error("Request error:", err) return } //var neInfo *dborm.NeInfo var nes []dborm.NeInfo _, err = dborm.XormGetAllNeInfo(&nes) if err != nil { log.Error("Failed to get all ne info:", err) services.ResponseInternalServerError500DatabaseOperationFailed(w) return } for _, ne := range nes { hostUri := fmt.Sprintf("http://%s:%v", ne.Ip, ne.Port) apiUri := fmt.Sprintf(UriAlarmsFmt, strings.ToLower(ne.NeType)) requestURI2NF := fmt.Sprintf("%s%s", hostUri, apiUri) log.Debug("requestURI2NF: Get ", requestURI2NF) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": config.GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). Get(requestURI2NF) if err != nil { log.Error("Failed to Get:", err) //services.ResponseInternalServerError500ProcessError(w, err) continue } body := response.Body() log.Debug("Request body:", string(body)) alarmArray := new([]Alarm) _ = json.Unmarshal(body, &alarmArray) valueJson, err := dborm.XormGetAAConfig() if err != nil { log.Error("Failed to XormGetAAConfig:", err) //services.ResponseInternalServerError500ProcessError(w, err) continue } session := xEngine.NewSession() defer session.Close() for _, alarmData := range *alarmArray { log.Debug("alarmData:", alarmData) // todo: clear alarm .... if alarmData.AlarmStatus == AlarmStatusClear { exist, err := session.Table("alarm"). Where("ne_type=? and ne_id=? and alarm_id=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.AlarmId). Exist() if err == nil || exist == false { log.Info("Not found active alarm: ne_id=%s, alarm_id=%s", alarmData.NeId, alarmData.AlarmId) continue } alarmData.ClearType = ClearTypeAutoClear alarmData.ClearTime.Valid = true tm, _ := time.Parse(time.RFC3339, alarmData.EventTime) log.Debugf("EventTime:%s tm:%d tm-datetime:%s", alarmData.EventTime, tm, tm.Local().Format(time.DateTime)) alarmData.ClearTime.Time = tm if IsNeedToAckAlarm(valueJson, &alarmData) == true { SetAlarmAckInfo(valueJson, &alarmData) log.Debug("alarmData:", alarmData) affected, err := session. Where("ne_type=? and ne_id=? and alarm_id=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.AlarmId). Cols("alarm_status", "clear_type", "clear_time", "ack_state", "ack_time", "ack_user"). Update(alarmData) if err != nil && affected <= 0 { log.Error("Failed to update alarm data:", err) //services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } } else { affected, err := session. Where("ne_type=? and ne_id=? and alarm_id=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.AlarmId). Cols("alarm_status", "clear_type", "clear_time"). Update(alarmData) if err != nil && affected <= 0 { log.Error("Failed to update alarm data:", err) //services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } } eventTime := global.GetFmtTimeString(time.RFC3339, alarmData.EventTime, time.DateTime) alarmLog := new(AlarmLog) alarmLog.NeType = alarmData.NeType alarmLog.NeId = alarmData.NeId alarmLog.AlarmSeq = alarmData.AlarmSeq alarmLog.AlarmId = alarmData.AlarmId alarmLog.AlarmCode = alarmData.AlarmCode alarmLog.AlarmStatus = alarmData.AlarmStatus alarmLog.EventTime = eventTime log.Debug("alarmLog:", alarmLog) affected, err := session.Insert(alarmLog) if err != nil && affected <= 0 { log.Error("Failed to insert alarm_log:", err) } // todo: PerceivedSeverity set color var severity string has, err := xEngine.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). //OrderBy("FIELD(orig_severity, 'Critical', 'Major', 'Minor', 'Warning', 'Event') ASC"). Asc("orig_severity"). Cols("orig_severity"). Limit(1). Get(&severity) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, eventTime=%s, severity=%s", alarmData.NeType, alarmData.NeId, alarmData.EventTime, severity) if has == true && severity > alarmData.OrigSeverity { // update exist record _, err := session.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=?", alarmData.NeType, alarmData.NeId, alarmData.EventTime). Update(&Alarm{PerceivedSeverity: severity}) if err != nil { log.Error("Failed to update alarm:", err) continue } } // for alarm forward time format alarmData.EventTime = eventTime } else { has, err := xEngine.Table("alarm"). Where("alarm_id=? and ne_type=? and ne_id=? and alarm_status=1", alarmData.AlarmId, alarmData.NeType, alarmData.NeId). Exist() if err == nil && has == true { log.Warn("Exist the same alarm") continue } var currentSeq string has, err = xEngine.Table("alarm"). Where("ne_type=? and ne_id=?", alarmData.NeType, alarmData.NeId). Desc("alarm_seq"). Cols("alarm_seq"). Limit(1). Get(¤tSeq) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, currentSeq=%s", alarmData.NeType, alarmData.NeId, currentSeq) if has == true { seq, _ := strconv.Atoi(currentSeq) if seq+1 > global.MaxInt32Number { alarmData.AlarmSeq = AlarmSeqBeginNumber } else { alarmData.AlarmSeq = seq + 1 } } else { alarmData.AlarmSeq = AlarmSeqBeginNumber } // todo: PerceivedSeverity set color var severity string has, err = xEngine.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=? and alarm_status=1", alarmData.NeType, alarmData.NeId, alarmData.EventTime). //OrderBy("FIELD(orig_severity, 'Critical', 'Major', 'Minor', 'Warning', 'Event') ASC"). Asc("orig_severity"). Cols("orig_severity"). Limit(1). Get(&severity) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debugf("neType=%s, neId=%s, eventTime=%s, severity=%s", alarmData.NeType, alarmData.NeId, alarmData.EventTime, severity) if has == false || severity == alarmData.OrigSeverity { alarmData.PerceivedSeverity = alarmData.OrigSeverity } else if severity > alarmData.OrigSeverity { alarmData.PerceivedSeverity = alarmData.OrigSeverity } else { alarmData.PerceivedSeverity = severity // update exist record _, err := session.Table("alarm"). Where("ne_type=? and ne_id=? and event_time=?", alarmData.NeType, alarmData.NeId, alarmData.EventTime). Update(&Alarm{PerceivedSeverity: alarmData.PerceivedSeverity}) if err != nil { log.Error("Failed to update alarm:", err) continue } } alarmData.ObjectUid = alarmData.NeId alarmData.ObjectType = "VNFM" alarmData.EventTime = global.GetFmtTimeString(time.RFC3339, alarmData.EventTime, time.DateTime) if IsNeedToAckAlarm(valueJson, &alarmData) == true { SetAlarmAckInfo(valueJson, &alarmData) } log.Debug("alarmData:", alarmData) affected, err := session.Insert(alarmData) if err == nil && affected > 0 { alarmLog := new(AlarmLog) alarmLog.NeType = alarmData.NeType alarmLog.NeId = alarmData.NeId alarmLog.AlarmSeq = alarmData.AlarmSeq alarmLog.AlarmId = alarmData.AlarmId alarmLog.AlarmCode = alarmData.AlarmCode alarmLog.AlarmStatus = alarmData.AlarmStatus alarmLog.EventTime = global.GetFmtTimeString(time.RFC3339, alarmData.EventTime, time.DateTime) log.Debug("alarmLog:", alarmLog) affected, err = session.Insert(alarmLog) if err != nil && affected <= 0 { log.Error("Failed to insert data:", err) //services.ResponseInternalServerError500DatabaseOperationFailed(w) continue } session.Commit() if config.GetYamlConfig().Alarm.ForwardAlarm { if err = AlarmEmailForward(&alarmData); err != nil { log.Error("Failed to AlarmEmailForward:", err) } if err = AlarmForwardBySMS(&alarmData); err != nil { log.Error("Failed to AlarmForwardBySMS:", err) } } } log.Warn("Failed to insert alarm data:", err) } } } services.ResponseStatusOK200Null(w) }