Merge branch 'lichang' into lite

2025-05-13 09:51:03 +08:00
parent 641612ab59 1ccbebce22
commit a0cffccbfa
16 changed files with 198 additions and 25 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # 版本发布日志

+## 2.2505.1-20250509
+
+- 修复 获取OMC状态版本号
+- 新增 根据网元显示特有菜单信息添加
+- 新增 网元授权添加用户容量列
+- 新增 网元状态记录内存/CPU/磁盘列表
+- 优化 CDR数据导出添加网元名称列
+- 新增 告警清除自定义的AlarmCode，告警LicenseCode/告警CPU内存磁盘
+
 ## 2.2504.4-20250430

 - 修复 捕获数据时跳过延迟超过1毫秒的包
--- a/build/database/lite/install/sys_i18n.sql
+++ b/build/database/lite/install/sys_i18n.sql
@@ -873,7 +873,7 @@ Minor=3
 Warning=4

 AddInfo: 告警补充信息
-cpuUseGt: CPU使用率大于， 范围0~100%
+cpuUseGt: CPU使用率大于， 范围0~100*CPU核心数
 memUseGt: 内存使用率大于， 范围0~100%
 diskUseGt: 磁盘使用率大于， 范围0~100%', 'Checks the memory/CPU/disk check health of the network element and sends alerts when thresholds are crossed.

@@ -891,7 +891,7 @@ Minor=3
 Warning=4

 AddInfo: Additional information on alarms
-cpuUseGt: CPU utilization greater than, range 0 to 100%
+cpuUseGt: CPU utilization is greater than, range 0~100*number of CPU cores
 memUseGt: Memory utilization greater than, range 0 to 100%
 diskUseGt: Disk utilization greater than, range 0 to 100%');
 INSERT INTO "sys_i18n" VALUES (754, 'job.ne_alarm_state_check_license', '网元告警-License到期检查', 'NE Alarm-License Expire Check');
--- a/build/database/lite/install/trace_task.sql
+++ b/build/database/lite/install/trace_task.sql
@@ -15,6 +15,7 @@ CREATE TABLE "trace_task" (
  "dst_ip" text(128),
  "create_by" text(50),
  "create_time" integer,
+  "title" text(255),
  "remark" text(500),
  "ne_list" text(32) NOT NULL,
  "notify_url" text(128) NOT NULL,
--- a/build/database/lite/upgrade/upg_trace_task.sql
+++ b/build/database/lite/upgrade/upg_trace_task.sql
@@ -0,0 +1,30 @@
+-- ----------------------------
+-- Table structure for trace_task
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS "trace_task" (
+  "id" integer NOT NULL,
+  "trace_id" text(16) NOT NULL,
+  "trace_type" text(2) NOT NULL,
+  "start_time" integer,
+  "end_time" integer,
+  "interfaces" text(255),
+  "imsi" text(16),
+  "msisdn" text(16),
+  "src_ip" text(128),
+  "dst_ip" text(128),
+  "create_by" text(50),
+  "create_time" integer,
+  "title" text(255),
+  "remark" text(500),
+  "ne_list" text(32) NOT NULL,
+  "notify_url" text(128) NOT NULL,
+  PRIMARY KEY ("id")
+);
+
+-- ----------------------------
+-- Indexes structure for table trace_task
+-- ----------------------------
+
+
+-- ADD COLUMN
+ALTER TABLE trace_task ADD COLUMN title text(255);
--- a/build/database/std/install/sys_i18n.sql
+++ b/build/database/std/install/sys_i18n.sql
@@ -767,7 +767,7 @@ INSERT INTO `sys_i18n` VALUES (749, 'dictType.cdr_sip_code_cause', 'IMS-Voice-SI
 INSERT INTO `sys_i18n` VALUES (750, 'job.backup_export_log', '备份-日志数据定期导出', 'Backup-Periodic export of Log Data');
 INSERT INTO `sys_i18n` VALUES (751, 'job.backup_export_log_remark', 'dataType: 类型支持 operate/login\nfileType: 文件类型 csv/xlsx\nhour: 数据时间从任务执行时间前的小时数', 'Backup-Periodic export of dataType: type support operate/login\nfileType: file type csv/xlsx\nhour: data time from the hour before the task execution time');
 INSERT INTO `sys_i18n` VALUES (752, 'job.ne_alarm_state_check_cmd', '网元告警-内存/CPU/磁盘检查', 'NE Alarm-Memory/CPU/Disk Checks');
-INSERT INTO `sys_i18n` VALUES (753, 'job.ne_alarm_state_check_cmd_remark', '检查网元的内存/CPU/磁盘检查健康状况，在出现过阈值时发出警报。\r\n\r\nAlarm type:\r\nCommunicationAlarm=1\r\nEquipmentAlarm=2\r\nProcessingFailure=3\r\nEnvironmentalAlarm=4\r\nQualityOfServiceAlarm=5\r\n\r\nSeverity:\r\nCritical=1\r\nMajor=2\r\nMinor=3\r\nWarning=4\r\n\r\nAddInfo: 告警补充信息\r\ncpuUseGt: CPU使用率大于， 范围0~100%\r\nmemUseGt: 内存使用率大于， 范围0~100%\r\ndiskUseGt: 磁盘使用率大于， 范围0~100%', 'Checks the memory/CPU/disk check health of the network element and sends alerts when thresholds are crossed.\n\nAlarm type:\nCommunicationAlarm=1\nEquipmentAlarm=2\nProcessingFailure=3\nEnvironmentalAlarm=4\nQualityOfServiceAlarm=5\n\nSeverity:\nCritical=1\nMajor=2\nMinor=3\nWarning=4\r\n\r\nAddInfo: Additional information on alarms\r\ncpuUseGt: CPU utilization greater than, range 0 to 100%\r\nmemUseGt: Memory utilization greater than, range 0 to 100%\r\ndiskUseGt: Disk utilization greater than, range 0 to 100%');
+INSERT INTO `sys_i18n` VALUES (753, 'job.ne_alarm_state_check_cmd_remark', '检查网元的内存/CPU/磁盘检查健康状况，在出现过阈值时发出警报。\r\n\r\nAlarm type:\r\nCommunicationAlarm=1\r\nEquipmentAlarm=2\r\nProcessingFailure=3\r\nEnvironmentalAlarm=4\r\nQualityOfServiceAlarm=5\r\n\r\nSeverity:\r\nCritical=1\r\nMajor=2\r\nMinor=3\r\nWarning=4\r\n\r\nAddInfo: 告警补充信息\r\ncpuUseGt: CPU使用率大于， 范围0~100*CPU核心数\r\nmemUseGt: 内存使用率大于， 范围0~100%\r\ndiskUseGt: 磁盘使用率大于， 范围0~100%', 'Checks the memory/CPU/disk check health of the network element and sends alerts when thresholds are crossed.\n\nAlarm type:\nCommunicationAlarm=1\nEquipmentAlarm=2\nProcessingFailure=3\nEnvironmentalAlarm=4\nQualityOfServiceAlarm=5\n\nSeverity:\nCritical=1\nMajor=2\nMinor=3\nWarning=4\r\n\r\nAddInfo: Additional information on alarms\r\ncpuUseGt: CPU utilization is greater than, range 0~100*number of CPU cores\r\nmemUseGt: Memory utilization greater than, range 0 to 100%\r\ndiskUseGt: Disk utilization greater than, range 0 to 100%');
 INSERT INTO `sys_i18n` VALUES (754, 'job.ne_alarm_state_check_license', '网元告警-License到期检查', 'NE Alarm-License Expire Check');
 INSERT INTO `sys_i18n` VALUES (755, 'job.ne_alarm_state_check_license_remark', '检查网元的License是否即将到期，在出现过阈值时发出警报。\r\n\r\nAlarm type:\r\nCommunicationAlarm=1\r\nEquipmentAlarm=2\r\nProcessingFailure=3\r\nEnvironmentalAlarm=4\r\nQualityOfServiceAlarm=5\r\n\r\nSeverity:\r\nCritical=1\r\nMajor=2\r\nMinor=3\r\nWarning=4\r\n\r\nAddInfo: 告警补充信息\r\ndayLt: 天数小于，默认30天', 'Checks if the network element is License is about to expire and sends an alert if a threshold is crossed.\n\nAlarm type:\nCommunicationAlarm=1\nEquipmentAlarm=2\nProcessingFailure=3\nEnvironmentalAlarm=4\nQualityOfServiceAlarm=5\n\nSeverity:\nCritical=1\nMajor=2\nMinor=3\nWarning=4\r\n\r\nAddInfo: Additional information on alarms\r\ndayLt: Days less than, default 30 days');

--- a/build/database/std/install/trace_task.sql
+++ b/build/database/std/install/trace_task.sql
@@ -16,6 +16,7 @@ CREATE TABLE `trace_task` (
  `dst_ip` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '目标地址IP',
  `create_by` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '创建者',
  `create_time` bigint DEFAULT '0' COMMENT '创建时间',
+  `title` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '' COMMENT '任务标题',
  `remark` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '' COMMENT '备注',
  `ne_list` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '网元列表 neType_neId 例如 UDM_001,AMF_001',
  `notify_url` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '信息数据通知回调地址UDP 例如udp:192.168.5.58:33033',
--- a/build/database/std/upgrade/upg_trace_task.sql
+++ b/build/database/std/upgrade/upg_trace_task.sql
@@ -0,0 +1,28 @@
+--
+-- Table structure for table `trace_task`
+--
+
+CREATE TABLE IF NOT EXISTS `trace_task` (
+  `id` int NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `trace_id` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '任务编号',
+  `trace_type` varchar(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '1-Interface,2-Device,3-UE',
+  `start_time` bigint DEFAULT '0' COMMENT '开始时间 毫秒',
+  `end_time` bigint DEFAULT '0' COMMENT '结束时间 毫秒',
+  `interfaces` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '' COMMENT '接口跟踪必须 例如 N8,N10',
+  `imsi` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户跟踪必须',
+  `msisdn` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户跟踪可选',
+  `src_ip` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '源地址IP',
+  `dst_ip` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '目标地址IP',
+  `create_by` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '创建者',
+  `create_time` bigint DEFAULT '0' COMMENT '创建时间',
+  `title` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '' COMMENT '任务标题',
+  `remark` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '' COMMENT '备注',
+  `ne_list` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '网元列表 neType_neId 例如 UDM_001,AMF_001',
+  `notify_url` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '信息数据通知回调地址UDP 例如udp:192.168.5.58:33033',
+  PRIMARY KEY (`id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='跟踪_任务';
+
+-- Dump completed on 2025-04-14 14:26:59
+
+-- ADD COLUMN
+ALTER TABLE `trace_task` ADD COLUMN `title` varchar(255) NULL COMMENT '任务标题';
--- a/features/fm/alarm.go
+++ b/features/fm/alarm.go
@@ -16,6 +16,7 @@ import (
 	"be.ems/lib/log"
 	"be.ems/lib/services"

+	"be.ems/src/framework/constants"
 	"be.ems/src/framework/resp"
 	"be.ems/src/framework/utils/date"
 	neDataModel "be.ems/src/modules/network_data/model"
@@ -23,6 +24,7 @@ import (
 	neFetchlink "be.ems/src/modules/network_element/fetch_link"
 	neModel "be.ems/src/modules/network_element/model"
 	neService "be.ems/src/modules/network_element/service"
+	traceService "be.ems/src/modules/trace/service"
 	wsService "be.ems/src/modules/ws/service"
 )

@@ -386,7 +388,7 @@ func alarmForward(v Alarm) {
 	}
 }

-// alarmEventNew 清除告警事件
+// alarmEventClear 清除告警事件
 func alarmEventClear(neInfo neModel.NeInfo, v Alarm) (neDataModel.AlarmEvent, error) {
 	alarmEventService := neDataService.NewAlarmEvent
 	// 检查网元告警ID是否唯一
@@ -469,6 +471,22 @@ func alarmEventNew(neInfo neModel.NeInfo, v Alarm) (neDataModel.AlarmEvent, erro
 	insertId := alarmEventService.Insert(alarmEvent)
 	if insertId > 0 {
 		alarmEvent.ID = insertId
+		// 网元重启后，清除活动告警
+		if v.AlarmCode == constants.ALARM_EVENT_REBOOT {
+			alarmService := neDataService.NewAlarm
+			rows := alarmService.Find(neDataModel.Alarm{
+				NeType:      neInfo.NeType,
+				NeId:        neInfo.NeId,
+				AlarmStatus: "1",
+			})
+			for _, v := range rows {
+				alarmService.AlarmClearByIds([]int64{v.ID}, "system")
+			}
+		}
+		// 网元重启后，有跟踪任务的需要重新补发启动任务
+		if v.AlarmCode == constants.ALARM_EVENT_REBOOT {
+			traceService.NewTraceTask.RunUnstopped()
+		}
 		return alarmEvent, nil
 	}
 	return neDataModel.AlarmEvent{}, fmt.Errorf("[%s %s] new alarm event fail", neInfo.NeType, neInfo.NeId)
--- a/src/framework/database/db/expand.go
+++ b/src/framework/database/db/expand.go
@@ -93,8 +93,16 @@ func processSQLFile(db *gorm.DB, filePath string) {
 		if strings.HasSuffix(line, ";") {
 			// 执行 SQL 语句
 			if err := db.Exec(sqlBuilder.String()).Error; err != nil {
-				log.Fatalln(err.Error())
-				return
+				errorStr := err.Error()
+				if strings.Contains(strings.ToLower(errorStr), "duplicate column") {
+					// 重复字段错误忽略
+					// log.Printf("Exec SQL: %s\n", line)
+					// log.Println(err.Error())
+				} else {
+					// 其他错误终止程序
+					log.Fatalln(errorStr)
+					return
+				}
 			}

 			sqlBuilder.Reset()
--- a/src/modules/crontask/processor/ne_alarm_state_check_cmd/ne_alarm_state_check_cmd.go
+++ b/src/modules/crontask/processor/ne_alarm_state_check_cmd/ne_alarm_state_check_cmd.go
@@ -3,6 +3,7 @@ package ne_alarm_state_check_cmd
 import (
 	"encoding/json"
 	"fmt"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -26,6 +27,8 @@ var NewProcessor = &NeAlarmStateCheckCMDProcessor{
 	alarmService:          neDataService.NewAlarm,
 	wsSendService:         wsService.NewWSSend,
 	count:                 0,
+	triggerMax:            4,
+	triggerCount:          0,
 }

 // NeAlarmStateCheckCMDProcessor 网元告警内存/CPU/磁盘检查
@@ -36,7 +39,8 @@ type NeAlarmStateCheckCMDProcessor struct {
 	alarmService          *neDataService.Alarm      // 告警信息服务
 	wsSendService         *wsService.WSSend         // ws发送服务
 	count                 int                       // 执行次数
-
+	triggerMax            int                       // 阈值连续触发次数大于才会产生告警
+	triggerCount          int                       // 阈值连续触发次数
 }

 // alarmParams 告警参数
@@ -47,7 +51,7 @@ type alarmParams struct {
 	SpecificProblem   string `json:"specificProblem"`   // Alarm Cause: CPU/Menory/Disk status received from target NE reaches the threshold
 	SpecificProblemID string `json:"specificProblemId"` // AC10100
 	AddInfo           string `json:"addInfo"`           // 告警补充信息
-	CPUUseGt          int64  `json:"cpuUseGt"`          // CPU使用率大于， 范围0~100%
+	CPUUseGt          int64  `json:"cpuUseGt"`          // CPU使用率大于， 范围0~100*CPU核心数
 	MemUseGt          int64  `json:"memUseGt"`          // 内存使用率大于， 范围0~100%
 	DiskUseGt         int64  `json:"diskUseGt"`         // 磁盘使用率大于， 范围0~100%

@@ -72,8 +76,9 @@ func (s *NeAlarmStateCheckCMDProcessor) Execute(data any) (any, error) {
 		return nil, fmt.Errorf("json params err: %v", err)
 	}
 	// 检查使用率
-	if params.CPUUseGt > 100 || params.CPUUseGt < 0 {
-		return nil, fmt.Errorf("cpuUseGt must be between 0 and 100")
+	numCPU := runtime.NumCPU()
+	if params.CPUUseGt > int64(numCPU*100) || params.CPUUseGt < 0 {
+		return nil, fmt.Errorf("cpuUseGt must be between 0 and 100 * NumCPU")
 	}
 	if params.MemUseGt > 100 || params.MemUseGt < 0 {
 		return nil, fmt.Errorf("memUseGt must be between 0 and 100")
@@ -137,6 +142,7 @@ func (s *NeAlarmStateCheckCMDProcessor) Execute(data any) (any, error) {
 			// 进行新增
 			newAlarm, err := s.alarmNew(neInfo, params)
 			params.AddInfo = addInfo // 恢复附加信息
+			s.triggerCount = 0       // 重置连续触发次数
 			if err != nil {
 				result[neTypeAndId] = err.Error()
 				continue
@@ -227,7 +233,10 @@ func (s NeAlarmStateCheckCMDProcessor) serverState(state map[string]any, cpuUseG
 		warnMsg = append(warnMsg, fmt.Sprintf("disk usage %.2f%%", sysDiskUsage))
 	}
 	if len(warnMsg) > 0 {
-		return fmt.Errorf("greater than %s", strings.Join(warnMsg, ", "))
+		s.triggerCount++
+		if s.triggerCount > s.triggerMax {
+			return fmt.Errorf("greater than %s", strings.Join(warnMsg, ", "))
+		}
 	}
 	return nil
 }
--- a/src/modules/crontask/processor/ne_alarm_state_check_license/ne_alarm_state_check_license.go
+++ b/src/modules/crontask/processor/ne_alarm_state_check_license/ne_alarm_state_check_license.go
@@ -120,8 +120,15 @@ func (s *NeAlarmStateCheckLicenseProcessor) Execute(data any) (any, error) {
 		}
 		// 未记录
 		if alarmStatus == "" {
+			addInfo := params.AddInfo
+			if params.AddInfo != "" {
+				params.AddInfo = params.AddInfo + ", " + err.Error()
+			} else {
+				params.AddInfo = err.Error()
+			}
 			// 进行新增
 			newAlarm, err := s.alarmNew(neInfo, params)
+			params.AddInfo = addInfo // 恢复附加信息
 			if err != nil {
 				result[neTypeAndId] = err.Error()
 				continue
@@ -150,7 +157,7 @@ func (s NeAlarmStateCheckLicenseProcessor) serverState(state map[string]any, day
 	}

 	// 计算距离天数
-	daysLeft := int64(time.Since(expireTime).Hours() / 24)
+	daysLeft := int64(time.Until(expireTime).Hours() / 24)
 	if daysLeft < dayLt {
 		return fmt.Errorf("license will expire in %d days", daysLeft)
 	}
--- a/src/modules/network_element/service/ne_version.go
+++ b/src/modules/network_element/service/ne_version.go
@@ -651,7 +651,18 @@ func (r NeVersion) operateRun(sshClient *ssh.ConnSSH, preinput map[string]string
 					}

 					// 命令终止符后继续执行命令
-					suffix := strings.HasSuffix(outputStr, "~]# ") || strings.LastIndex(outputStr, "~# ") != -1 || strings.HasSuffix(outputStr, "~$ ")
+					// "~]# ":麒麟, "~]$ ":欧拉, "~# ":NXP, "~$ ":Ubuntu
+					suffixStr := []string{"~]# ", "~]$ ", "~# ", "~$ "}
+					suffix := false
+					for _, v := range suffixStr {
+						if strings.HasSuffix(outputStr, v) {
+							suffix = true
+							break
+						}
+					}
+					if !suffix {
+						suffix = strings.LastIndex(outputStr, "~# ") != -1
+					}
 					if len(cmdStrArr) > 0 && suffix {
 						if firstRead {
 							firstRead = false
--- a/src/modules/trace/model/trace_task.go
+++ b/src/modules/trace/model/trace_task.go
@@ -14,6 +14,7 @@ type TraceTask struct {
 	DstIp      string `json:"dstIp"  gorm:"column:dst_ip"`                            // 目标地址IP
 	CreateBy   string `json:"createBy"  gorm:"column:create_by"`                      // 创建者
 	CreateTime int64  `json:"createTime"  gorm:"column:create_time"`                  // 创建时间
+	Title      string `json:"title"  gorm:"column:title" binding:"required"`          // 任务标题
 	Remark     string `json:"remark"  gorm:"column:remark"`                           // 备注
 	NeList     string `json:"neList"  gorm:"column:ne_list" binding:"required"`       // 网元列表 neType_neId 例如 UDM_001,AMF_001
 	NotifyUrl  string `json:"notifyUrl"  gorm:"column:notify_url"`                    // 信息数据通知回调地址UDP 例如udp:192.168.5.58:33033
--- a/src/modules/trace/repository/trace_task.go
+++ b/src/modules/trace/repository/trace_task.go
@@ -29,6 +29,9 @@ func (r TraceTask) SelectByPage(query map[string]string) ([]model.TraceTask, int
 	if v, ok := query["msisdn"]; ok && v != "" {
 		tx = tx.Where("msisdn like ?", fmt.Sprintf("%s%%", v))
 	}
+	if v, ok := query["title"]; ok && v != "" {
+		tx = tx.Where("title like ?", fmt.Sprintf("%s%%", v))
+	}
 	if v, ok := query["startTime"]; ok && v != "" {
 		if len(v) == 10 {
 			v = fmt.Sprintf("%s000", v)
@@ -112,6 +115,20 @@ func (r TraceTask) SelectByIds(ids []int64) []model.TraceTask {
 	return rows
 }

+// SelectByUnstopped 查询未停止的任务补发
+func (r TraceTask) SelectByUnstopped() []model.TraceTask {
+	rows := []model.TraceTask{}
+	tx := db.DB("").Model(&model.TraceTask{})
+	// 构建查询条件
+	tx = tx.Where("end_time > ?", time.Now().UnixMilli())
+	// 查询数据
+	if err := tx.Find(&rows).Error; err != nil {
+		logger.Errorf("query find err => %v", err.Error())
+		return rows
+	}
+	return rows
+}
+
 // Insert 新增信息 返回新增数据ID
 func (r TraceTask) Insert(param model.TraceTask) int64 {
 	if param.CreateBy != "" {
--- a/src/modules/trace/service/trace_task.go
+++ b/src/modules/trace/service/trace_task.go
@@ -177,6 +177,19 @@ func (r TraceTask) FindById(id int64) model.TraceTask {

 // Insert 新增信息
 func (r TraceTask) Insert(task model.TraceTask) error {
+	if err := r.createTaskToNe(&task, false); err != nil {
+		return err
+	}
+	// 插入数据库
+	insertId := r.traceTaskRepository.Insert(task)
+	if insertId <= 0 {
+		return fmt.Errorf("insert task error")
+	}
+	return nil
+}
+
+// CreateTaskToNe 创建任务到网元
+func (r TraceTask) createTaskToNe(task *model.TraceTask, ignoreErr bool) error {
 	// 跟踪配置是否开启
 	host, port, err := r.traceNotify()
 	if err != nil {
@@ -193,6 +206,7 @@ func (r TraceTask) Insert(task model.TraceTask) error {
 	task.TraceId = fmt.Sprint(traceId)

 	// 发送任务给网元
+	errNe := []string{}
 	for _, neTypeID := range neList {
 		neTypeIDArr := strings.Split(neTypeID, "_")
 		if len(neTypeIDArr) != 2 {
@@ -204,18 +218,28 @@ func (r TraceTask) Insert(task model.TraceTask) error {
 			logger.Warnf("ne type id is not exist")
 			continue
 		}
-		if err := r.createTaskToNe(neInfo, task); err != nil {
-			logger.Errorf("task to %s error: %s", neTypeID, err.Error())
-			return fmt.Errorf("task to %s error: %s", neTypeID, err.Error())
+		if err := r.traceNeTask(neInfo, *task); err != nil {
+			logger.Errorf("ne type id is %s to %s error: %s", task.TraceId, neTypeID, err.Error())
+			errNe = append(errNe, neTypeID)
+			continue
 		}
 	}
-
-	// 插入数据库
-	insertId := r.traceTaskRepository.Insert(task)
-	if insertId <= 0 {
-		return fmt.Errorf("insert task error")
+	if len(errNe) <= 0 && !ignoreErr {
+		return nil
 	}
-	return nil
+	// 移除任务
+	for _, neTypeID := range neList {
+		neTypeIDArr := strings.Split(neTypeID, "_")
+		if len(neTypeIDArr) != 2 {
+			continue
+		}
+		neInfo := neService.NewNeInfo.FindByNeTypeAndNeID(neTypeIDArr[0], neTypeIDArr[1])
+		if neInfo.NeType != neTypeIDArr[0] || neInfo.IP == "" {
+			continue
+		}
+		neFetchlink.NeTraceDelete(neInfo, task.TraceId)
+	}
+	return fmt.Errorf("task to ne error: %s", strings.Join(errNe, ","))
 }

 // traceNotify 网元通知地址
@@ -234,8 +258,8 @@ func (r TraceTask) traceNotify() (string, int64, error) {
 	return host, port, nil
 }

-// createTaskToNe 网元创建任务
-func (r TraceTask) createTaskToNe(neInfo neModel.NeInfo, task model.TraceTask) error {
+// traceNeTask 网元创建任务
+func (r TraceTask) traceNeTask(neInfo neModel.NeInfo, task model.TraceTask) error {
 	data := model.TraceReq{
 		TraceId:   parse.Number(task.TraceId),
 		NotifyUrl: task.NotifyUrl,
@@ -305,3 +329,11 @@ func (r TraceTask) DeleteByIds(ids []int64) (int64, error) {
 	// 删除信息失败！
 	return 0, fmt.Errorf("delete fail")
 }
+
+// RunUnstopped 启动跟踪未停止的任务
+func (r TraceTask) RunUnstopped() {
+	tasks := r.traceTaskRepository.SelectByUnstopped()
+	for _, task := range tasks {
+		r.createTaskToNe(&task, true)
+	}
+}
--- a/src/modules/trace/service/trace_task_udp_data.go
+++ b/src/modules/trace/service/trace_task_udp_data.go
@@ -27,7 +27,7 @@ func traceHandler(data []byte) (*TraceMsgToOamTraceData, error) {
 	if err != nil {
 		return decodeData, err
 	}
-	fmt.Printf("TraceHandler get oamData: %s,%+v\n, payload=len(%d,%d)", decodeData.TimestampStr, decodeData.NfTraceMsg, decodeData.TracePayloadLen, len(decodeData.TracePayload))
+	// fmt.Printf("TraceHandler get oamData: %s,%+v\n, payload=len(%d,%d)", decodeData.TimestampStr, decodeData.NfTraceMsg, decodeData.TracePayloadLen, len(decodeData.TracePayload))
 	// Return parsed message and payload
 	if len(decodeData.TracePayload) != int(decodeData.TracePayloadLen) {
 		return decodeData, fmt.Errorf("trace payload is bad, len=%d, shall be:%d", len(decodeData.TracePayload), int(decodeData.TracePayloadLen))
@@ -106,6 +106,7 @@ func decodeTraceData(data []byte) (*TraceMsgToOamTraceData, error) {
 		return nil, err
 	}
 	msg.SrcIpStr = net.IP(srcIp).String()
+
 	// 7. Parse DstIp (4 bytes IPv4)
 	dstIp := make([]byte, 4)
 	if flag&0x20 != 0 {