feat: 备份网元Log文件到网管支持ftp转存

This commit is contained in:
TsMask
2025-10-13 17:31:23 +08:00
parent 344da83d10
commit 2587723dea
8 changed files with 153 additions and 5 deletions

View File

@@ -72,7 +72,7 @@ INSERT INTO `sys_dict_data` VALUES (1064, 1064, 'dictData.cdr_sip_code_cause.202
INSERT INTO `sys_dict_data` VALUES (1065, 1065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1066, 1066, 'config.neData.backupDataFTP', '备份网元数据-同步FTP服务', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1067, 1067, 'config.neData.backupDataFTPRemark', '请通过系统页面进行设置FTP信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-系统日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1069, 1069, 'job.backup_export_log_remark', 'dataType: 类型支持 operate/login\nfileType: 文件类型 csv/xlsx\nhour: 数据时间从任务执行时间前的小时数', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1070, 1070, 'menu.ueUser.onlineIMS', 'IMS在线用户', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (1071, 1071, 'menu.ueUser.onlineUE', 'UE在线信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -784,6 +784,7 @@ INSERT INTO `sys_dict_data` VALUES (2273, 2273, 'dictData.cdr_cause_smsc.7', 'SI
INSERT INTO `sys_dict_data` VALUES (2274, 2274, 'dictData.cdr_cause_smsc.8', 'SIP_MT_NOT_HAVE_RP_ACK', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (2275, 2275, 'dictData.cdr_cause_smsc.9', 'SIP_MT_RP_ACK_ERROR', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (2276, 2276, 'dictData.cdr_cause_smsc.10', 'SIP_CAMEL_ERROR', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (2277, 2277, 'job.backup_export_log_ne', '备份-网元日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-- multi-tenancy
INSERT INTO `sys_dict_data` VALUES (11000, 11000, 'menu.security.tenant', '租户管理', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);

View File

@@ -72,7 +72,7 @@ INSERT INTO `sys_dict_data` VALUES (3064, 3064, 'dictData.cdr_sip_code_cause.202
INSERT INTO `sys_dict_data` VALUES (3065, 3065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3066, 3066, 'config.neData.backupDataFTP', 'Backup NE Data - Sync Data FTP Service', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3067, 3067, 'config.neData.backupDataFTPRemark', 'Please set the FTP information through the system page.', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of System Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3069, 3069, 'job.backup_export_log_remark', 'Backup-Periodic export of dataType: type support operate/login\nfileType: file type csv/xlsx\nhour: data time from the hour before the task execution time', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3070, 3070, 'menu.ueUser.onlineIMS', 'IMS Online User', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
INSERT INTO `sys_dict_data` VALUES (3071, 3071, 'menu.ueUser.onlineUE', 'UE PDU Session', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -784,6 +784,7 @@ INSERT INTO `sys_dict_data` VALUES (4273, 4273, 'dictData.cdr_cause_smsc.7', 'Me
INSERT INTO `sys_dict_data` VALUES (4274, 4274, 'dictData.cdr_cause_smsc.8', 'SMSC not receive RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (4275, 4275, 'dictData.cdr_cause_smsc.9', 'SMSC receive wrong RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (4276, 4276, 'dictData.cdr_cause_smsc.10', 'CAMEL process error', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
INSERT INTO `sys_dict_data` VALUES (4277, 4277, 'job.backup_export_log_ne', 'Backup-Periodic export of NE Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-- 多租户
INSERT INTO `sys_dict_data` VALUES (14000, 14000, 'menu.security.tenant', 'Tenant Management', 'i18n_en', '', '', '1', 'supervisor', 1705550000000, '', 0, '');

View File

@@ -63,5 +63,6 @@ INSERT INTO `sys_job` VALUES (31, 'job.backup_export_udm', 'SYSTEM', 'backup_exp
INSERT INTO `sys_job` VALUES (32, 'job.backup_export_cdr', 'SYSTEM', 'backup_export_cdr', '{\"dataType\":[\"ims\",\"smf\",\"sgwc\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_cdr_remark');
INSERT INTO `sys_job` VALUES (33, 'job.backup_export_log', 'SYSTEM', 'backup_export_log', '{\"dataType\":[\"operate\",\"login\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_log_remark');
INSERT INTO `sys_job` VALUES (34, 'job.backup_export_kpi', 'SYSTEM', 'backup_export_kpi', '{\"dataType\":[\"ims\",\"amf\",\"udm\",\"smf\",\"pcf\",\"upf\",\"mme\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_kpi_remark');
INSERT INTO `sys_job` VALUES (35, 'job.backup_export_log_ne', 'SYSTEM', 'backup_export_log_ne', '', '0 59 23 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, '');
SET FOREIGN_KEY_CHECKS = 1;

View File

@@ -85,7 +85,7 @@ REPLACE INTO `sys_dict_data` VALUES (1064, 1064, 'dictData.cdr_sip_code_cause.20
REPLACE INTO `sys_dict_data` VALUES (1065, 1065, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1066, 1066, 'config.neData.backupDataFTP', '备份网元数据-同步FTP服务', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1067, 1067, 'config.neData.backupDataFTPRemark', '请通过系统页面进行设置FTP信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1068, 1068, 'job.backup_export_log', '备份-系统日志数据定期导出', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1069, 1069, 'job.backup_export_log_remark', 'dataType: 类型支持 operate/login\nfileType: 文件类型 csv/xlsx\nhour: 数据时间从任务执行时间前的小时数', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1070, 1070, 'menu.ueUser.onlineIMS', 'IMS在线用户', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (1071, 1071, 'menu.ueUser.onlineUE', 'UE在线信息', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);

View File

@@ -85,7 +85,7 @@ REPLACE INTO `sys_dict_data` VALUES (3064, 3064, 'dictData.cdr_sip_code_cause.20
REPLACE INTO `sys_dict_data` VALUES (3066, 3066, 'dictData.cdr_sip_code_cause.0', 'Call failure for other reason', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3066, 3066, 'config.neData.backupDataFTP', 'Backup NE Data - Sync Data FTP Service', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3067, 3067, 'config.neData.backupDataFTPRemark', 'Please set the FTP information through the system page.', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3068, 3068, 'job.backup_export_log', 'Backup-Periodic export of System Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3069, 3069, 'job.backup_export_log_remark', 'Backup-Periodic export of dataType: type support operate/login\nfileType: file type csv/xlsx\nhour: data time from the hour before the task execution time', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3070, 3070, 'menu.ueUser.onlineIMS', 'IMS Online User', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
REPLACE INTO `sys_dict_data` VALUES (3071, 3071, 'menu.ueUser.onlineUE', 'UE PDU Session', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
@@ -792,6 +792,7 @@ REPLACE INTO `sys_dict_data` VALUES (4273, 4273, 'dictData.cdr_cause_smsc.7', 'M
REPLACE INTO `sys_dict_data` VALUES (4274, 4274, 'dictData.cdr_cause_smsc.8', 'SMSC not receive RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
REPLACE INTO `sys_dict_data` VALUES (4275, 4275, 'dictData.cdr_cause_smsc.9', 'SMSC receive wrong RP-ACK pkg(cscf->smsc)', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
REPLACE INTO `sys_dict_data` VALUES (4276, 4276, 'dictData.cdr_cause_smsc.10', 'CAMEL process error', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, '');
REPLACE INTO `sys_dict_data` VALUES (4277, 4277, 'job.backup_export_log_ne', 'Backup-Periodic export of NE Log Data', 'i18n_en', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL);
-- 多租户
REPLACE INTO `sys_dict_data` VALUES (14000, 14000, 'menu.security.tenant', 'Tenant Management', 'i18n_en', '', '', '1', 'supervisor', 1705550000000, '', 0, '');

View File

@@ -67,5 +67,6 @@ REPLACE INTO `sys_job` VALUES (31, 'job.backup_export_udm', 'SYSTEM', 'backup_ex
REPLACE INTO `sys_job` VALUES (32, 'job.backup_export_cdr', 'SYSTEM', 'backup_export_cdr', '{\"dataType\":[\"ims\",\"smf\",\"sgwc\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_cdr_remark');
REPLACE INTO `sys_job` VALUES (33, 'job.backup_export_log', 'SYSTEM', 'backup_export_log', '{\"dataType\":[\"operate\",\"login\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_log_remark');
REPLACE INTO `sys_job` VALUES (34, 'job.backup_export_kpi', 'SYSTEM', 'backup_export_kpi', '{\"dataType\":[\"ims\",\"amf\",\"udm\",\"smf\",\"pcf\",\"upf\",\"mme\",\"smsc\"],\"fileType\":\"xlsx\",\"hour\":1}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, 'job.backup_export_kpi_remark');
REPLACE INTO `sys_job` VALUES (35, 'job.backup_export_log_ne', 'SYSTEM', 'backup_export_log_ne', '', '0 59 23 * * ?', '3', '0', '1', '1', 'system', 1698478134839, 'supervisor', 1745481169354, '');
SET FOREIGN_KEY_CHECKS = 1;

View File

@@ -0,0 +1,140 @@
package backup_export_log_ne
import (
"fmt"
"path/filepath"
"strings"
"time"
"be.ems/src/framework/cron"
"be.ems/src/framework/database/redis"
"be.ems/src/framework/logger"
"be.ems/src/framework/ssh"
neDataService "be.ems/src/modules/network_data/service"
neModel "be.ems/src/modules/network_element/model"
neService "be.ems/src/modules/network_element/service"
)
var NewProcessor = &BackupExportLogNEProcessor{
count: 0,
backupService: neDataService.NewBackup,
neInfoService: neService.NewNeInfo,
logFilePath: "/var/log",
}
// BackupExportLogNEProcessor 备份导出日志任务处理
type BackupExportLogNEProcessor struct {
count int // 执行次数
backupService *neDataService.Backup // 备份相关服务
neInfoService *neService.NeInfo // 网元信息服务
logFilePath string // 日志文件目录
}
func (s *BackupExportLogNEProcessor) Execute(data any) (any, error) {
s.count++ // 执行次数加一
options := data.(cron.JobData)
sysJob := options.SysJob
logger.Infof("重复:%v 任务ID:%s 执行次数:%d", options.Repeat, sysJob.JobID, s.count)
// 返回结果,用于记录执行结果
result := map[string]any{
"count": s.count,
}
// 分布式锁,防止多个任务同时执行
lockKey := fmt.Sprintf("processor:backup_export_log_ne:%s", sysJob.JobID)
if ok := redis.SetNX("", lockKey, time.Minute); !ok {
return nil, cron.ErrTaskRunning
}
defer redis.Del("", lockKey)
// 时间目录
timeDir := time.Now().Format("20060102150405")
neList := s.neInfoService.Find(neModel.NeInfo{}, false, false)
for _, neInfo := range neList {
if neInfo.IP == "" {
continue
}
// 网元主机的SSH客户端
sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId)
if err != nil {
result[neInfo.NeName] = err.Error()
continue
}
defer sshClient.Close()
// 网元主机的SSH客户端进行文件传输
sftpClient, err := sshClient.NewClientSFTP()
if err != nil {
result[neInfo.NeName] = err.Error()
continue
}
defer sftpClient.Close()
// 备份导出日志
if neInfo.NeType == "IMS" {
imsDirArr := [...]string{"icscf", "bgcf", "mmtel", "pcscf", "scscf", "iwf", "bsf", "ismc"}
for _, dirPath := range imsDirArr {
logFilePathIMS := filepath.ToSlash(filepath.Join(s.logFilePath, "ims", dirPath))
output := s.backup(timeDir, logFilePathIMS, neInfo, sshClient, sftpClient)
for k, v := range output {
result[k+"-"+dirPath] = v
}
}
} else {
output := s.backup(timeDir, s.logFilePath, neInfo, sshClient, sftpClient)
for k, v := range output {
result[k] = v
}
}
}
// 返回结果,用于记录执行结果
return result, nil
}
// backup 备份导出日志
func (s *BackupExportLogNEProcessor) backup(timeDir string, logFilePath string, neInfo neModel.NeInfo, sshClient *ssh.ConnSSH, sftpClient *ssh.SSHClientSFTP) map[string]string {
var result = map[string]string{}
// 获取文件列表
search := strings.ToLower(neInfo.NeType)
if neInfo.NeType == "IMS" {
search = ""
}
rows, err := ssh.FileList(sshClient, logFilePath, search)
if err != nil {
result[neInfo.NeName] = err.Error()
return result
}
// 计算今天00:00:00的时间戳
today := time.Now()
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, today.Location())
todayStartTimestamp := todayStart.Unix()
for _, row := range rows {
if row.ModifiedTime < todayStartTimestamp || row.FileType != "file" {
continue
}
// 网元日志文件路径
neFilePath := fmt.Sprintf("%s/%s", logFilePath, row.FileName)
// 复制到备份目录
neDirPath := fmt.Sprintf("%s/%s", search, row.FileName)
if neInfo.NeType == "IMS" {
neDirPath = strings.Replace(neFilePath, s.logFilePath, "", 1)
}
localFilePath := filepath.Join(s.backupService.BACKUP_DIR, "/log/ne_log", timeDir, neDirPath)
if err := sftpClient.CopyFileRemoteToLocal(neFilePath, localFilePath); err != nil {
result[neInfo.NeName+"-copy"] = err.Error()
continue
}
// 上传到FTP服务器
if err := s.backupService.FTPPushFile(localFilePath, "log"); err != nil {
result[neInfo.NeName+"-copy-ftp"] = "ok, ftp err:" + err.Error()
continue
}
}
return result
}

View File

@@ -6,6 +6,7 @@ import (
processorBackupExportCDR "be.ems/src/modules/crontask/processor/backup_export_cdr"
processorBackupExportKPI "be.ems/src/modules/crontask/processor/backup_export_kpi"
processorBackupExportLog "be.ems/src/modules/crontask/processor/backup_export_log"
processorBackupExportLogNE "be.ems/src/modules/crontask/processor/backup_export_log_ne"
processorBackupExportUDM "be.ems/src/modules/crontask/processor/backup_export_udm"
processorBackupRemoveFile "be.ems/src/modules/crontask/processor/backup_remove_file"
"be.ems/src/modules/crontask/processor/delExpiredNeBackup"
@@ -73,8 +74,10 @@ func InitCronQueue() {
cron.CreateQueue("backup_export_udm", processorBackupExportUDM.NewProcessor)
// 备份-导出CDR数据
cron.CreateQueue("backup_export_cdr", processorBackupExportCDR.NewProcessor)
// 备份-导出Log数据
// 备份-导出系统Log数据
cron.CreateQueue("backup_export_log", processorBackupExportLog.NewProcessor)
// 备份-导出KPI数据
cron.CreateQueue("backup_export_kpi", processorBackupExportKPI.NewProcessor)
// 备份-导出网元Log数据
cron.CreateQueue("backup_export_log_ne", processorBackupExportLogNE.NewProcessor)
}