feat: 备份网元Log文件到网管支持ftp转存

database/install/sys_dict_data1_i18n_zh.sql

@@ -72,7 +72,7 @@ INSERT INTO `sys_dict_data` VALUES (1064, 1064, 'dictData.cdr_sip_code_cause.202
 INSERT INTO `sys_dict_data` VALUES (1065, 1065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (1066, 1066, 'config.neData.backupDataFTP', '备份网元数据-同步FTP服务', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (1067, 1067, 'config.neData.backupDataFTPRemark', '请通过系统页面进行设置FTP信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-INSERT INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
+INSERT INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-系统日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (1069, 1069, 'job.backup_export_log_remark', 'dataType: 类型支持 operate/login\nfileType: 文件类型 csv/xlsx\nhour: 数据时间从任务执行时间前的小时数', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (1070, 1070, 'menu.ueUser.onlineIMS', 'IMS在线用户', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (1071, 1071, 'menu.ueUser.onlineUE', 'UE在线信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -784,6 +784,7 @@ INSERT INTO `sys_dict_data` VALUES (2273, 2273, 'dictData.cdr_cause_smsc.7', 'SI
 INSERT INTO `sys_dict_data` VALUES (2274, 2274, 'dictData.cdr_cause_smsc.8', 'SIP_MT_NOT_HAVE_RP_ACK', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 INSERT INTO `sys_dict_data` VALUES (2275, 2275, 'dictData.cdr_cause_smsc.9', 'SIP_MT_RP_ACK_ERROR', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 INSERT INTO `sys_dict_data` VALUES (2276, 2276, 'dictData.cdr_cause_smsc.10', 'SIP_CAMEL_ERROR', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
+INSERT INTO `sys_dict_data` VALUES (2277, 2277, 'job.backup_export_log_ne', '备份-网元日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 
 -- multi-tenancy
 INSERT INTO `sys_dict_data` VALUES (11000, 11000, 'menu.security.tenant', '租户管理', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);

database/install/sys_dict_data2_i18n_en.sql

@@ -72,7 +72,7 @@ INSERT INTO `sys_dict_data` VALUES (3064, 3064, 'dictData.cdr_sip_code_cause.202
 INSERT INTO `sys_dict_data` VALUES (3065, 3065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (3066, 3066, 'config.neData.backupDataFTP', 'Backup NE Data - Sync Data FTP Service', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (3067, 3067, 'config.neData.backupDataFTPRemark', 'Please set the FTP information through the system page.', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-INSERT INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
+INSERT INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of System Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (3069, 3069, 'job.backup_export_log_remark', 'Backup-Periodic export of dataType: type support operate/login\nfileType: file type csv/xlsx\nhour: data time from the hour before the task execution time', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (3070, 3070, 'menu.ueUser.onlineIMS', 'IMS Online User', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 INSERT INTO `sys_dict_data` VALUES (3071, 3071, 'menu.ueUser.onlineUE', 'UE PDU Session', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -784,6 +784,7 @@ INSERT INTO `sys_dict_data` VALUES (4273, 4273, 'dictData.cdr_cause_smsc.7', 'Me
 INSERT INTO `sys_dict_data` VALUES (4274, 4274, 'dictData.cdr_cause_smsc.8', 'SMSC not receive RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 INSERT INTO `sys_dict_data` VALUES (4275, 4275, 'dictData.cdr_cause_smsc.9', 'SMSC receive wrong RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 INSERT INTO `sys_dict_data` VALUES (4276, 4276, 'dictData.cdr_cause_smsc.10', 'CAMEL process error', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
+INSERT INTO `sys_dict_data` VALUES (4277, 4277, 'job.backup_export_log_ne', 'Backup-Periodic export of NE Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 
 -- 多租户
 INSERT INTO `sys_dict_data` VALUES (14000, 14000, 'menu.security.tenant', 'Tenant Management', 'i18n_en', '', '', '1', 'supervisor', 1705550000000, '', 0, '');

database/install/sys_job.sql

@@ -63,5 +63,6 @@ INSERT INTO `sys_job` VALUES (31, 'job.backup_export_udm', 'SYSTEM', 'backup_exp
 INSERT INTO `sys_job` VALUES (32, 'job.backup_export_cdr', 'SYSTEM', 'backup_export_cdr', '{\"dataType\":[\"ims\",\"smf\",\"sgwc\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_cdr_remark');
 INSERT INTO `sys_job` VALUES (33, 'job.backup_export_log', 'SYSTEM', 'backup_export_log', '{\"dataType\":[\"operate\",\"login\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_log_remark');
 INSERT INTO `sys_job` VALUES (34, 'job.backup_export_kpi', 'SYSTEM', 'backup_export_kpi', '{\"dataType\":[\"ims\",\"amf\",\"udm\",\"smf\",\"pcf\",\"upf\",\"mme\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_kpi_remark');
+INSERT INTO `sys_job` VALUES (35, 'job.backup_export_log_ne', 'SYSTEM', 'backup_export_log_ne', '', '0 59 23 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, '');
 
 SET FOREIGN_KEY_CHECKS = 1;

database/upgrade/upg_sys_dict_data1_i18n_zh.sql

@@ -85,7 +85,7 @@ REPLACE INTO `sys_dict_data` VALUES (1064, 1064, 'dictData.cdr_sip_code_cause.20
 REPLACE INTO `sys_dict_data` VALUES (1065, 1065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (1066, 1066, 'config.neData.backupDataFTP', '备份网元数据-同步FTP服务', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (1067, 1067, 'config.neData.backupDataFTPRemark', '请通过系统页面进行设置FTP信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-REPLACE INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
+REPLACE INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-系统日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (1069, 1069, 'job.backup_export_log_remark', 'dataType: 类型支持 operate/login\nfileType: 文件类型 csv/xlsx\nhour: 数据时间从任务执行时间前的小时数', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (1070, 1070, 'menu.ueUser.onlineIMS', 'IMS在线用户', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (1071, 1071, 'menu.ueUser.onlineUE', 'UE在线信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);

database/upgrade/upg_sys_dict_data2_i18n_en.sql

@@ -85,7 +85,7 @@ REPLACE INTO `sys_dict_data` VALUES (3064, 3064, 'dictData.cdr_sip_code_cause.20
 REPLACE INTO `sys_dict_data` VALUES (3066, 3066, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (3066, 3066, 'config.neData.backupDataFTP', 'Backup NE Data - Sync Data FTP Service', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (3067, 3067, 'config.neData.backupDataFTPRemark', 'Please set the FTP information through the system page.', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-REPLACE INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
+REPLACE INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of System Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (3069, 3069, 'job.backup_export_log_remark', 'Backup-Periodic export of dataType: type support operate/login\nfileType: file type csv/xlsx\nhour: data time from the hour before the task execution time', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (3070, 3070, 'menu.ueUser.onlineIMS', 'IMS Online User', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 REPLACE INTO `sys_dict_data` VALUES (3071, 3071, 'menu.ueUser.onlineUE', 'UE PDU Session', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -792,6 +792,7 @@ REPLACE INTO `sys_dict_data` VALUES (4273, 4273, 'dictData.cdr_cause_smsc.7', 'M
 REPLACE INTO `sys_dict_data` VALUES (4274, 4274, 'dictData.cdr_cause_smsc.8', 'SMSC not receive RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 REPLACE INTO `sys_dict_data` VALUES (4275, 4275, 'dictData.cdr_cause_smsc.9', 'SMSC receive wrong RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
 REPLACE INTO `sys_dict_data` VALUES (4276, 4276, 'dictData.cdr_cause_smsc.10', 'CAMEL process error', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
+REPLACE INTO `sys_dict_data` VALUES (4277, 4277, 'job.backup_export_log_ne', 'Backup-Periodic export of NE Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
 
 -- 多租户
 REPLACE INTO `sys_dict_data` VALUES (14000, 14000, 'menu.security.tenant', 'Tenant Management', 'i18n_en', '', '', '1', 'supervisor', 1705550000000, '', 0, '');

database/upgrade/upg_sys_job.sql

@@ -67,5 +67,6 @@ REPLACE INTO `sys_job` VALUES (31, 'job.backup_export_udm', 'SYSTEM', 'backup_ex
 REPLACE INTO `sys_job` VALUES (32, 'job.backup_export_cdr', 'SYSTEM', 'backup_export_cdr', '{\"dataType\":[\"ims\",\"smf\",\"sgwc\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_cdr_remark');
 REPLACE INTO `sys_job` VALUES (33, 'job.backup_export_log', 'SYSTEM', 'backup_export_log', '{\"dataType\":[\"operate\",\"login\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_log_remark');
 REPLACE INTO `sys_job` VALUES (34, 'job.backup_export_kpi', 'SYSTEM', 'backup_export_kpi', '{\"dataType\":[\"ims\",\"amf\",\"udm\",\"smf\",\"pcf\",\"upf\",\"mme\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_kpi_remark');
+REPLACE INTO `sys_job` VALUES (35, 'job.backup_export_log_ne', 'SYSTEM', 'backup_export_log_ne', '', '0 59 23 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, '');
 
 SET FOREIGN_KEY_CHECKS = 1;

src/modules/crontask/processor/backup_export_log_ne/backup_export_log_ne.go

@@ -0,0 +1,140 @@
+package backup_export_log_ne
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"be.ems/src/framework/cron"
+	"be.ems/src/framework/database/redis"
+	"be.ems/src/framework/logger"
+	"be.ems/src/framework/ssh"
+	neDataService "be.ems/src/modules/network_data/service"
+	neModel "be.ems/src/modules/network_element/model"
+	neService "be.ems/src/modules/network_element/service"
+)
+
+var NewProcessor = &BackupExportLogNEProcessor{
+	count:         0,
+	backupService: neDataService.NewBackup,
+	neInfoService: neService.NewNeInfo,
+	logFilePath:   "/var/log",
+}
+
+// BackupExportLogNEProcessor 备份导出日志任务处理
+type BackupExportLogNEProcessor struct {
+	count         int                   // 执行次数
+	backupService *neDataService.Backup // 备份相关服务
+	neInfoService *neService.NeInfo     // 网元信息服务
+	logFilePath   string                // 日志文件目录
+}
+
+func (s *BackupExportLogNEProcessor) Execute(data any) (any, error) {
+	s.count++ // 执行次数加一
+	options := data.(cron.JobData)
+	sysJob := options.SysJob
+	logger.Infof("重复:%v 任务ID:%s  执行次数:%d", options.Repeat, sysJob.JobID, s.count)
+	// 返回结果，用于记录执行结果
+	result := map[string]any{
+		"count": s.count,
+	}
+
+	// 分布式锁，防止多个任务同时执行
+	lockKey := fmt.Sprintf("processor:backup_export_log_ne:%s", sysJob.JobID)
+	if ok := redis.SetNX("", lockKey, time.Minute); !ok {
+		return nil, cron.ErrTaskRunning
+	}
+	defer redis.Del("", lockKey)
+
+	// 时间目录
+	timeDir := time.Now().Format("20060102150405")
+
+	neList := s.neInfoService.Find(neModel.NeInfo{}, false, false)
+	for _, neInfo := range neList {
+		if neInfo.IP == "" {
+			continue
+		}
+
+		// 网元主机的SSH客户端
+		sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId)
+		if err != nil {
+			result[neInfo.NeName] = err.Error()
+			continue
+		}
+		defer sshClient.Close()
+		// 网元主机的SSH客户端进行文件传输
+		sftpClient, err := sshClient.NewClientSFTP()
+		if err != nil {
+			result[neInfo.NeName] = err.Error()
+			continue
+		}
+		defer sftpClient.Close()
+
+		// 备份导出日志
+		if neInfo.NeType == "IMS" {
+			imsDirArr := [...]string{"icscf", "bgcf", "mmtel", "pcscf", "scscf", "iwf", "bsf", "ismc"}
+			for _, dirPath := range imsDirArr {
+				logFilePathIMS := filepath.ToSlash(filepath.Join(s.logFilePath, "ims", dirPath))
+				output := s.backup(timeDir, logFilePathIMS, neInfo, sshClient, sftpClient)
+				for k, v := range output {
+					result[k+"-"+dirPath] = v
+				}
+			}
+		} else {
+			output := s.backup(timeDir, s.logFilePath, neInfo, sshClient, sftpClient)
+			for k, v := range output {
+				result[k] = v
+			}
+		}
+
+	}
+
+	// 返回结果，用于记录执行结果
+	return result, nil
+}
+
+// backup 备份导出日志
+func (s *BackupExportLogNEProcessor) backup(timeDir string, logFilePath string, neInfo neModel.NeInfo, sshClient *ssh.ConnSSH, sftpClient *ssh.SSHClientSFTP) map[string]string {
+	var result = map[string]string{}
+
+	// 获取文件列表
+	search := strings.ToLower(neInfo.NeType)
+	if neInfo.NeType == "IMS" {
+		search = ""
+	}
+	rows, err := ssh.FileList(sshClient, logFilePath, search)
+	if err != nil {
+		result[neInfo.NeName] = err.Error()
+		return result
+	}
+
+	// 计算今天00:00:00的时间戳（秒）
+	today := time.Now()
+	todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, today.Location())
+	todayStartTimestamp := todayStart.Unix()
+	for _, row := range rows {
+		if row.ModifiedTime < todayStartTimestamp || row.FileType != "file" {
+			continue
+		}
+		// 网元日志文件路径
+		neFilePath := fmt.Sprintf("%s/%s", logFilePath, row.FileName)
+		// 复制到备份目录
+		neDirPath := fmt.Sprintf("%s/%s", search, row.FileName)
+		if neInfo.NeType == "IMS" {
+			neDirPath = strings.Replace(neFilePath, s.logFilePath, "", 1)
+		}
+		localFilePath := filepath.Join(s.backupService.BACKUP_DIR, "/log/ne_log", timeDir, neDirPath)
+		if err := sftpClient.CopyFileRemoteToLocal(neFilePath, localFilePath); err != nil {
+			result[neInfo.NeName+"-copy"] = err.Error()
+			continue
+		}
+		// 上传到FTP服务器
+		if err := s.backupService.FTPPushFile(localFilePath, "log"); err != nil {
+			result[neInfo.NeName+"-copy-ftp"] = "ok, ftp err:" + err.Error()
+			continue
+		}
+	}
+
+	return result
+}

src/modules/crontask/processor/processor.go

@@ -6,6 +6,7 @@ import (
 	processorBackupExportCDR "be.ems/src/modules/crontask/processor/backup_export_cdr"
 	processorBackupExportKPI "be.ems/src/modules/crontask/processor/backup_export_kpi"
 	processorBackupExportLog "be.ems/src/modules/crontask/processor/backup_export_log"
+	processorBackupExportLogNE "be.ems/src/modules/crontask/processor/backup_export_log_ne"
 	processorBackupExportUDM "be.ems/src/modules/crontask/processor/backup_export_udm"
 	processorBackupRemoveFile "be.ems/src/modules/crontask/processor/backup_remove_file"
 	"be.ems/src/modules/crontask/processor/delExpiredNeBackup"
@@ -73,8 +74,10 @@ func InitCronQueue() {
 	cron.CreateQueue("backup_export_udm", processorBackupExportUDM.NewProcessor)
 	// 备份-导出CDR数据
 	cron.CreateQueue("backup_export_cdr", processorBackupExportCDR.NewProcessor)
-	// 备份-导出Log数据
+	// 备份-导出系统Log数据
 	cron.CreateQueue("backup_export_log", processorBackupExportLog.NewProcessor)
 	// 备份-导出KPI数据
 	cron.CreateQueue("backup_export_kpi", processorBackupExportKPI.NewProcessor)
+	// 备份-导出网元Log数据
+	cron.CreateQueue("backup_export_log_ne", processorBackupExportLogNE.NewProcessor)
 }