package main import ( "bytes" "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/http" "os" "reflect" "strings" "time" "ems.agt/lib/dborm" "ems.agt/lib/global" "ems.agt/lib/log" "github.com/go-resty/resty/v2" "github.com/shirou/gopsutil/process" // cron "github.com/go-co-op/gocron" "github.com/jasonlvhit/gocron" "github.com/robfig/cron/v3" ) type TaskFunc struct { } var Alarms []Alarm func initCronTasks() { taskFuncs := TaskFunc{} vf := reflect.ValueOf(&taskFuncs) for _, t := range taskSet.Tasks { log.Trace("Task:", t) if strings.ToLower(t.Status) == TaskStatusInactive { continue } //Call function by function name taskFunc := vf.MethodByName(t.Do).Interface() switch t.Unit { case "Seconds": gocron.Every(t.Interval).Seconds().DoSafely(taskFunc, t.Uri, t.Params, t.Body) case "Minutes": gocron.Every(t.Interval).Minutes().DoSafely(taskFunc, t.Uri, t.Params, t.Body) case "Hour": if t.At == "" { gocron.Every(t.Interval).Hour().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { ats := strings.Split(t.At, ",") for _, at := range ats { gocron.Every(t.Interval).Day().At(at).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } } case "Hours": gocron.Every(t.Interval).Hours().DoSafely(taskFunc, t.Uri, t.Params, t.Body) case "Days": if t.At == "" { gocron.Every(t.Interval).Days().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { ats := strings.Split(t.At, ",") for _, at := range ats { gocron.Every(t.Interval).Days().At(at).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } } case "Day": if t.At == "" { gocron.Every(t.Interval).Day().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { ats := strings.Split(t.At, ",") for _, at := range ats { gocron.Every(t.Interval).Day().At(at).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } } case "Weeks": if t.At == "" { gocron.Every(t.Interval).Weeks().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Weeks().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Monday": if t.At == "" { gocron.Every(t.Interval).Monday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Monday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Tuesday": if t.At == "" { gocron.Every(t.Interval).Tuesday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Tuesday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Wednesday": if t.At == "" { gocron.Every(t.Interval).Wednesday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Wednesday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Thursday": if t.At == "" { gocron.Every(t.Interval).Thursday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Thursday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Friday": if t.At == "" { gocron.Every(t.Interval).Friday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Friday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Saturday": if t.At == "" { gocron.Every(t.Interval).Saturday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Saturday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } case "Sunday": if t.At == "" { gocron.Every(t.Interval).Sunday().DoSafely(taskFunc, t.Uri, t.Params, t.Body) } else { gocron.Every(t.Interval).Sunday().At(t.At).DoSafely(taskFunc, t.Uri, t.Params, t.Body) } default: log.Error("Error config:", t) } } // remove, clear and next_rull _, time := gocron.NextRun() log.Info(time) <-gocron.Start() } func initCronTabs() { taskFuncs := TaskFunc{} vf := reflect.ValueOf(&taskFuncs) job := cron.New( // cron.WithSeconds(), // 添加秒级别支持,默认支持最小粒度为分钟 ) for _, c := range taskSet.Crontabs { log.Debug(c.Name) if c.Status == TaskStatusInactive { continue } taskFunc := vf.MethodByName(c.Do).Interface() if taskFunc != nil { job.AddJob(c.Tab, TaskJob{taskFunc, c.Uri, c.Params, c.Body, &taskFuncs}) } } job.Start() } type TaskJob struct { Do interface{} Uri string Params string Body string T *TaskFunc } func (j TaskJob) Run() { log.Tracef("Uri:%s, Params:%s Body:%s", j.Uri, j.Params, j.Body) err := j.T.CallTaskFunc(j.Do, j.Uri, j.Params, j.Body) if err != nil { log.Error("call func error:", err) } } func (t *TaskFunc) CallTaskFunc(jobFunc interface{}, params ...interface{}) error { f := reflect.ValueOf(jobFunc) if len(params) != f.Type().NumIn() { err := global.ErrParamsNotAdapted log.Fatal(err) return err } in := make([]reflect.Value, len(params)) for k, param := range params { in[k] = reflect.ValueOf(param) } f.Call(in) return nil } func (t *TaskFunc) TaskHelloWorld(uri, params, body string) { log.Infof("Hello, world! uri:%s, params:%s Date: %s", uri, params, time.Now().Format("2006-01-02")) } func (t *TaskFunc) TaskWithParams(a int, b string) { log.Trace(a, b) } func (t *TaskFunc) TaskDeleteExpiredRecord(uri, params, body string) { log.Debug("TaskDeleteExpiredRecord processing... ") var response *resty.Response requestURI := fmt.Sprintf("%s?%s", uri, params) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: DELETE ", requestURL) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). Delete(requestURL) if err != nil { log.Error("Failed to delete:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } func (t *TaskFunc) TaskUpdateTable(uri, params, body string) { log.Debug("TaskUpdateTable processing... ") var response *resty.Response requestURI := fmt.Sprintf("%s?%s", uri, params) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) // reqBody, err := json.Marshal(body) // if err != nil { // log.Error("Failed to Marshal:", err) // } log.Debug("requestURL: Put ", requestURL) log.Trace("body:", body) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Put(requestURL) if err != nil { log.Error("Failed to put:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } func (t *TaskFunc) GetTableNameFromUri(uri string) string { sa := global.SplitString(uri, "/") n := len(sa) if n > 0 { return sa[n-1] } return "" } func (t *TaskFunc) TaskDBBackupCSVGetBySQL(uri, params, body string) { log.Debug("TaskDBBackupCSVGetBySQL processing... ") var response *resty.Response tableName := t.GetTableNameFromUri(uri) filePath := fmt.Sprintf("%s/%s-%s.csv", GetYamlConfig().Database.Backup, tableName, time.Now().Local().Format(global.DateData)) pa := fmt.Sprintf(params, filePath) requestURI := fmt.Sprintf("%s?%s", uri, pa) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: Get ", requestURL) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). Get(requestURL) if err != nil { log.Error("Failed to Get:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Error("response body:", string(response.Body())) } } type CpuUsage struct { NfCpuUsage uint16 `json:"nfCpuUsage"` SysCpuUsage uint16 `json:"sysCpuUsage"` } type MemUsage struct { TotalMem uint32 `json:"totalMem"` NfUsedMem uint32 `json:"nfUsedMem"` SysMemUsage uint16 `json:"sysMemUsage"` } type PartitionInfo struct { Total uint32 `json:"total"` // MB Used uint32 `json:"used"` // MB } type DiskSpace struct { PartitionNum uint8 `json:"partitionNum"` PartitionInfo []PartitionInfo `json:"partitionInfo"` } type SystemState struct { Version string `json:"version"` Capability uint32 `json:"capability"` SerialNum string `json:"serialNum"` ExpiryDate string `json:"expiryDate"` //Timestamp string `json:"timestamp"` CpuUsage CpuUsage `json:"cpuUsage"` MemUsage MemUsage `json:"memUsage"` DiskSpace DiskSpace `json:"diskSpace"` } func (t *TaskFunc) TaskHandShakeToNF(uri, params, body string) { log.Debug("TaskHandShakeToNF processing... ") var nes []NeInfo _, err := XormGetAllNeInfo(&nes) if err != nil { log.Error("Failed to get all ne info:", err) } for _, ne := range nes { requestURI := fmt.Sprintf(uri, strings.ToLower(ne.NeType)) if params != "" { requestURI = fmt.Sprintf("%s?%s", requestURI, params) } requestURL := fmt.Sprintf("http://%s:%s%s", ne.Ip, ne.Port, requestURI) log.Debug("requestURL: Get", requestURL) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). Get(requestURL) if err != nil { log.Error("Failed to Get:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) state := new(SystemState) _ = json.Unmarshal(response.Body(), &state) neState := new(NeState) neState.NeType = ne.NeType neState.NeId = ne.NeId neState.Version = state.Version neState.Capability = state.Capability neState.SerialNum = state.SerialNum neState.ExpiryDate = state.ExpiryDate cu, _ := json.Marshal(state.CpuUsage) neState.CpuUsage = string(cu) mu, _ := json.Marshal(state.MemUsage) neState.MemUsage = string(mu) ds, _ := json.Marshal(state.DiskSpace) neState.DiskSpace = string(ds) log.Debug("neState:", neState) _, err := XormInsertNeState(neState) if err != nil { log.Error("Failed to insert ne_state:", err) } default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } func (t *TaskFunc) TaskExportCmFromNF(uri, params, body string) { log.Debug("TaskExportCmFromNF processing... ") var nes []NeInfo _, err := XormGetAllNeInfo(&nes) if err != nil { log.Error("Failed to get all ne info:", err) } for _, ne := range nes { requestURI := fmt.Sprintf(uri, strings.ToLower(ne.NeType)) if params != "" { paramsUri := fmt.Sprintf(params, ne.NeId) requestURI = fmt.Sprintf("%s?%s", requestURI, paramsUri) } requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: Get", requestURL) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). // SetOutput("./export/test.zip"). Get(requestURL) if err != nil { log.Error("Failed to Get:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("Export NF cm sucessfully") //defer response.Body.Close() //fileName := "./export/" + response.Header().Get("User-File") //os.Rename("./export/test.zip", fileName) default: log.Error("Failed to export NF cm") } } } func (t *TaskFunc) TaskImportCmToNF(uri, params, body string) { log.Debug("TaskImportCmToNF processing... ") file, err := os.OpenFile("udm-sz_01-etc-20230612001524.zip", os.O_RDONLY, os.ModePerm) if err != nil { log.Error("Failed to OpenFile:", err) return } byteBody := &bytes.Buffer{} //body = new(bytes.Buffer) writer := multipart.NewWriter(byteBody) part, err := writer.CreateFormFile("file", "./temp/"+file.Name()) if err != nil { log.Error("Failed to CreateFormFile:", err) return } _, err = io.Copy(part, file) // if err != nil { log.Error("Failed to Copy:", err) return } writer.Close() requestURI := fmt.Sprintf("%s?%s", uri, params) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) client := resty.New() response, err := client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": writer.FormDataContentType()}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to Post:", err) return } switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("Import NF cm sucessfully") default: log.Error("response code:", response.StatusCode()) } return } func (t *TaskFunc) TaskGenMeasureThresholdAlarm(uri, params, body string) { log.Info("TaskGenMeasureThresholdAlarm processing...") // get measurethreshold table var measureThresholds []MeasureThreshold _, err := XormGetMeasureThreshold("measure_threshold", "status='Active'", &measureThresholds) if err != nil { log.Error("Failed to get measure_threshold:", err) return } for _, m := range measureThresholds { measureData, err := XormGetMeasureDataOneByKpi(m.KpiSet) if err != nil { log.Error("Failed to get measure_data:", err) continue } log.Debug("measureData:", measureData) alarmDefine, err := XormGetAlarmDefine(m.AlarmId) if err != nil { log.Error("Failed to get alarm_define:", err) continue } else if alarmDefine == nil { log.Error("Not found data from alarm_define") continue } log.Debug("alarmDefine:", alarmDefine) // sql := fmt.Sprintf("select * from alarm where alarm_id = '%s' and ne_type='%s' and ne_id = '%s' order by event_time desc limit 1", // m.AlarmId, m.NeType, measureData.RmUid) // alarm, err := XormGetDataBySQL(sql) // if err != nil { // log.Error("Failed to get alarm:", err) // continue // } // log.Debug("alarm:", *alarm) // kpi pm non-exceed, check if exist alarm if measureData.Value <= m.Threshold && m.AlarmFlag == true { // if len(*alarm) == 0 || (*alarm)[0]["alarm_status"] == AlarmStatusClearString { // continue // } // clear alarm, todo var alarmSeq int = 1 // insert alarm var pvFlag string sql := fmt.Sprintf("select * from ne_info where ne_type='%s' and rm_uid='%s' limit 1", m.NeType, measureData.RmUid) neInfo, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get ne_info:", err) continue } log.Debug("neInfo:", neInfo) if len(*neInfo) > 0 { pvFlag = (*neInfo)[0]["pv_flag"] } if pvFlag == "" { pvFlag = "VNF" } locationInfo := fmt.Sprintf("PM.%s: value=%v, threshold=%v", m.KpiSet, measureData.Value, m.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: m.AlarmId, NeId: measureData.RmUid, NeType: m.NeType, NeName: measureData.NeName, Province: GetYamlConfig().OMC.Province, PVFlag: pvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusClear, OrigSeverity: m.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: alarmDefine.SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, m.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) // todo ... update measure_threshold set alarm_flag=false m.AlarmFlag = false _, err := XormUpdateTableById(m.Id, "measure_threshold", m, "alarm_flag") if err != nil { log.Error("Failed to update measure_threshold:", err) continue } default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } else if measureData.Value > m.Threshold && m.AlarmFlag == false { var alarmSeq int = 1 // insert alarm var pvFlag string sql := fmt.Sprintf("select * from ne_info where ne_type='%s' and rm_uid='%s' limit 1", m.NeType, measureData.RmUid) neInfo, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get ne_info:", err) continue } log.Debug("neInfo:", neInfo) if len(*neInfo) > 0 { pvFlag = (*neInfo)[0]["pv_flag"] } if pvFlag == "" { pvFlag = "VNF" } locationInfo := fmt.Sprintf("PM.%s: value=%v,threshold=%v", m.KpiSet, measureData.Value, m.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: m.AlarmId, NeId: measureData.RmUid, NeType: m.NeType, NeName: measureData.NeName, Province: GetYamlConfig().OMC.Province, PVFlag: pvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusActive, OrigSeverity: m.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: alarmDefine.SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, m.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) // todo ... update measure_threshold set alarm_flag=true m.AlarmFlag = true _, err := XormUpdateTableById(m.Id, "measure_threshold", m, "alarm_flag") if err != nil { log.Error("Failed to update measure_threshold:", err) continue } default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } } func (t *TaskFunc) TaskGenMeasureReportTimeoutAlarm(uri, params, body string) { log.Info("TaskGenMeasureReportTimeoutAlarm processing...") var measureTasks []dborm.MeasureTask _, err := dborm.XormGetActiveMeasureTask(&measureTasks) if err != nil { log.Error("Failed to GetActiveMeasureTask: ", err) return } log.Trace("measureTasks:", measureTasks) for _, measureTask := range measureTasks { if len(measureTask.NeIds) == 0 { var neInfos []dborm.NeInfo err := dborm.XormGetNeInfoByNeType(measureTask.NeType, &neInfos) if err != nil { log.Error("Failed to dborm.XormGetNeInfoByNeType:", err) continue } for _, neInfo := range neInfos { measureTask.NeIds = append(measureTask.NeIds, neInfo.NeId) } } log.Debug("measureTask.NeIds:", measureTask.NeIds) for _, neId := range measureTask.NeIds { var err error neInfo, err := dborm.XormGetNeInfo(measureTask.NeType, neId) if err != nil { log.Error("Failed to dborm.XormGetNeInfo:", err) continue } if neInfo == nil { err := errors.New(fmt.Sprintf("not found target NE neType=%s, neId=%s", measureTask.NeType, neId)) log.Error(err) continue } measureData, err := XormGetMeasureDataLastOne(measureTask.NeType, neInfo.RmUID, measureTask.Id) if err != nil { log.Error("Failed to get measure_data:", err) continue } log.Debug("measureData:", measureData) var alarmId string = "RJHXEMSPM10201" if params != "" { alarmId = params } alarmDefine, err := XormGetAlarmDefine(alarmId) if err != nil { log.Error("Failed to get alarm_define:", err) continue } else if alarmDefine == nil { log.Error("Not found data from alarm_define") continue } log.Debug("alarmDefine:", alarmDefine) log.Debugf("measureData.StartTime:%s measureTask.StartTime:%s measureTask.CreateTime:%s", measureData.StartTime, measureTask.StartTime, measureTask.CreateTime) var startTime string if measureData.StartTime != "" { startTime = measureData.StartTime layout := "2006-01-02 15:04:05 -0700 MST" startTime = global.GetFmtTimeString(layout, startTime, time.DateTime) } else if measureTask.StartTime != "" { startTime = measureTask.StartTime } else if measureTask.CreateTime != "" { startTime = measureTask.CreateTime layout := "2006-01-02 15:04:05 -0700 MST" startTime = global.GetFmtTimeString(layout, startTime, time.DateTime) } log.Debugf("original startTime:%s", startTime) seconds, err := global.GetSecondsSinceDatetime(startTime) if err != nil { log.Error("Failed to GetSecondsSinceDatetime:", err) continue } log.Debugf("startTime:%s seconds:%d", startTime, seconds) var isTimeout bool = false var specificProblem string switch measureTask.GranulOption { case "15M": if seconds > ((15 + 10) * 60) { isTimeout = true specificProblem = fmt.Sprintf(alarmDefine.SpecificProblem, 15, 10) } case "30M": if seconds > ((30 + 15) * 60) { isTimeout = true specificProblem = fmt.Sprintf(alarmDefine.SpecificProblem, 30, 15) } case "60M": if seconds > ((60 + 15) * 60) { isTimeout = true specificProblem = fmt.Sprintf(alarmDefine.SpecificProblem, 60, 15) } } if isTimeout { // if len(*alarm) == 0 || (*alarm)[0]["alarm_status"] == AlarmStatusClearString { // continue // } // todo var alarmSeq int = 1 locationInfo := fmt.Sprintf("PM.taskId=%d: neType=%s, neId=%s", measureTask.Id, neInfo.NeType, neInfo.NeId) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: alarmId, NeId: neInfo.RmUID, NeType: measureTask.NeType, NeName: neInfo.NeName, Province: neInfo.Province, PVFlag: neInfo.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusActive, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: specificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, neInfo.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) // todo ... update measure_threshold set alarm_flag=false default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } else { var alarmSeq int = 1 locationInfo := fmt.Sprintf("PM.taskId=%d: neType=%s, neId=%s", measureTask.Id, neInfo.NeType, neInfo.NeId) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: alarmId, NeId: neInfo.RmUID, NeType: measureTask.NeType, NeName: neInfo.NeName, Province: neInfo.Province, PVFlag: neInfo.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusClear, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: specificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, neInfo.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) // todo ... update measure_threshold set alarm_flag=false default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } } } func (t *TaskFunc) TaskGenLicenseAlarm(uri, params, body string) { log.Info("TaskGenLicenseAlarm processing...") var nes []NeInfo _, err := XormGetAllNeInfo(&nes) if err != nil { log.Error("Failed to get all ne info:", err) } for _, ne := range nes { log.Debug("ne:", ne) sql := fmt.Sprintf("select * from ne_state where ne_type = '%s' and ne_id = '%s' order by timestamp desc limit 1", ne.NeType, ne.NeId) neState, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get ne_state:", err) continue } log.Debug("neState:", *neState) if len(*neState) == 0 { log.Errorf("Not found ne_state neType:%s, neId:%s", ne.NeType, ne.NeId) continue } if params == "" { params = "RJHXEMSCM10100" } alarmDefine, err := XormGetAlarmDefine(params) if err != nil { log.Error("Failed to get alarm_define:", err) continue } else if alarmDefine == nil { log.Error("Not found data from alarm_define") continue } log.Debug("alarmDefine:", alarmDefine) sql = fmt.Sprintf("select * from alarm where alarm_id = '%s' and ne_type='%s' and ne_id = '%s' order by event_time desc limit 1", alarmDefine.AlarmId, ne.NeType, ne.RmUID) alarm, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debug("alarm:", *alarm) expiryDate := (*neState)[0]["expiry_date"] days := global.GetDayDuration(expiryDate, time.Now().Format(time.DateOnly)) log.Debugf("expiryDate:%s days:%d", expiryDate, days) // kpi pm non-exceed, check if exist alarm if days > int64(alarmDefine.Threshold) { if len(*alarm) == 0 || (*alarm)[0]["alarm_status"] == AlarmStatusClearString { continue } // clear alarm, todo var alarmSeq int = 1 SpecificProblem := fmt.Sprintf(alarmDefine.SpecificProblem, alarmDefine.Threshold) locationInfo := fmt.Sprintf("CM.License: expiry date=%s,threshold=%v", expiryDate, alarmDefine.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: params, NeId: ne.RmUID, NeType: ne.NeType, NeName: ne.NeName, Province: ne.Province, PVFlag: ne.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusClear, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, ne.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } else { var alarmSeq int = 1 if len(*alarm) > 0 && (*alarm)[0]["alarm_status"] == AlarmStatusActiveString { log.Info("License alarm has exist") continue } SpecificProblem := fmt.Sprintf(alarmDefine.SpecificProblem, alarmDefine.Threshold) locationInfo := fmt.Sprintf("CM.License: expiry date=%s,threshold=%v", expiryDate, alarmDefine.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: params, NeId: ne.RmUID, NeType: ne.NeType, NeName: ne.NeName, Province: ne.Province, PVFlag: ne.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusActive, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, ne.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } } func (t *TaskFunc) TaskGenNeStateAlarm(uri, params, body string) { log.Info("TaskGenNeStateAlarm processing...") var nes []NeInfo _, err := XormGetAllNeInfo(&nes) if err != nil { log.Error("Failed to get all ne info:", err) } for _, ne := range nes { log.Debug("ne:", ne) sql := fmt.Sprintf("select * from ne_state where ne_type = '%s' and ne_id = '%s' order by timestamp desc limit 1", ne.NeType, ne.NeId) neState, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get ne_state:", err) continue } log.Debug("neState:", *neState) if params == "" { params = "RJHXEMSSM10000" } alarmDefine, err := XormGetAlarmDefine(params) if err != nil { log.Error("Failed to get alarm_define:", err) continue } else if alarmDefine == nil { log.Error("Not found data from alarm_define") continue } log.Debug("alarmDefine:", alarmDefine) sql = fmt.Sprintf("select * from alarm where alarm_id = '%s' and ne_type='%s' and ne_id = '%s' order by event_time desc limit 1", alarmDefine.AlarmId, ne.NeType, ne.RmUID) alarm, err := XormGetDataBySQL(sql) if err != nil { log.Error("Failed to get alarm:", err) continue } log.Debug("alarm:", *alarm) var timestamp string if len(*neState) == 0 { log.Infof("Not found ne_state neType:%s, neId:%s", ne.NeType, ne.NeId) timestamp = ne.UpdateTime } else { timestamp = (*neState)[0]["timestamp"] } // 解析日期时间字符串为时间对象 seconds, err := global.GetSecondsSinceDatetime(timestamp) if err != nil { log.Error("Failed to GetSecondsSinceDatetime:", err) continue } log.Debugf("timestamp:%s seconds:%d", timestamp, seconds) if seconds <= alarmDefine.Threshold { if len(*alarm) == 0 || (*alarm)[0]["alarm_status"] == AlarmStatusClearString { continue } // clear alarm, todo var alarmSeq int = 1 SpecificProblem := fmt.Sprintf(alarmDefine.SpecificProblem, alarmDefine.Threshold) locationInfo := fmt.Sprintf("SystemManagement.State: NE heartbeat timestamp=%s,threshold=%v", timestamp, alarmDefine.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: params, NeId: ne.RmUID, NeType: ne.NeType, NeName: ne.NeName, Province: ne.Province, PVFlag: ne.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusClear, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, ne.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } else { var alarmSeq int = 1 if len(*alarm) > 0 && (*alarm)[0]["alarm_status"] == AlarmStatusActiveString { log.Info("System state alarm has exist") continue } SpecificProblem := fmt.Sprintf(alarmDefine.SpecificProblem, alarmDefine.Threshold) locationInfo := fmt.Sprintf("SystemManagement.State: NE heartbeat timestamp=%s,threshold=%v", timestamp, alarmDefine.Threshold) alarmData := &Alarm{ AlarmSeq: alarmSeq, AlarmId: params, NeId: ne.RmUID, NeType: ne.NeType, NeName: ne.NeName, Province: ne.Province, PVFlag: ne.PvFlag, AlarmCode: alarmDefine.AlarmCode, AlarmTitle: alarmDefine.AlarmTitle, AlarmType: alarmDefine.AlarmType, AlarmStatus: AlarmStatusActive, OrigSeverity: alarmDefine.OrigSeverity, ObjectUid: alarmDefine.ObjectUid, ObjectName: alarmDefine.ObjectName, ObjectType: alarmDefine.ObjectType, LocationInfo: locationInfo, SpecificProblem: SpecificProblem, SpecificProblemID: alarmDefine.SpecificProblemId, AddInfo: alarmDefine.AddInfo, EventTime: time.Now().Local().Format(time.RFC3339), } alarmArray := &[]Alarm{*alarmData} body, _ := json.Marshal(alarmArray) log.Debug("body: ", string(body)) var response *resty.Response requestURI := fmt.Sprintf(uri, ne.NeType) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } } type ProcInfo struct { Name string `json:"name"` Pid int32 `json:"pid"` StartTime string `json:"startTime"` } var procList = []string{"restagent", "crontask", "4a_agent", "nbi_agent", "nbi_alarm", "mysqld"} var procInfos = make([]ProcInfo, len(procList)) func init() { for i, procName := range procList { procInfo := &ProcInfo{ Name: procName, Pid: 0, StartTime: "", } procInfos[i] = *procInfo } } func (t *TaskFunc) TaskWriteSystemLog(uri, params, body string) { processes, err := process.Processes() if err != nil { log.Error("Failed to get processes:", err) return } for _, proc := range processes { name, err := proc.Name() if err != nil { log.Error("Failed to get process name:", err) continue } for i, procInfo := range procInfos { if name == procInfo.Name { pid := proc.Pid timestamp, _ := proc.CreateTime() userName, _ := proc.Username() startTime := time.Unix((timestamp / 1000), 0).Format(time.DateTime) newProcInfo := &ProcInfo{ Name: procInfo.Name, Pid: pid, StartTime: startTime, } log.Trace("newProcInfo:", *newProcInfo) log.Trace("procInfo:", procInfo) if *newProcInfo != procInfo { procInfos[i] = *newProcInfo var operation string if newProcInfo.Pid != 0 { operation = "start" } else { operation = "stop" } sysLogs := &[]dborm.SystemLog{} sysLog := &dborm.SystemLog{ User: userName, ProcessName: newProcInfo.Name, ProcessID: newProcInfo.Pid, Operation: operation, ProcessTime: newProcInfo.StartTime, LogTime: time.Now().Format(time.DateTime), } var response *resty.Response requestURI := fmt.Sprintf("%s?%s", uri, params) requestURL := fmt.Sprintf("%s%s", yamlConfig.OMC.HostUri, requestURI) *sysLogs = append(*sysLogs, *sysLog) data := make(map[string]interface{}) data["system_log"] = *sysLogs body, err := json.Marshal(data) log.Trace("data:", data) log.Debug("requestURL: POST ", requestURL) client := resty.New() response, err = client.R(). EnableTrace(). SetHeaders(map[string]string{"User-Agent": GetDefaultUserAgent()}). SetHeaders(map[string]string{"Content-Type": "application/json;charset=UTF-8"}). SetBody(body). Post(requestURL) if err != nil { log.Error("Failed to post:", err) } log.Debug("StatusCode: ", response.StatusCode()) switch response.StatusCode() { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) default: log.Debug("response body:", string(response.Body())) body := new(map[string]interface{}) _ = json.Unmarshal(response.Body(), &body) } } } } } }