feat: 新增调度任务-导出自定义KPI数据备份

This commit is contained in:
TsMask
2025-09-17 16:53:45 +08:00
parent 9d3eb2a289
commit 397947f0b9
4 changed files with 678 additions and 10 deletions

View File

@@ -22,18 +22,20 @@ import (
)
var NewProcessor = &BackupExportKPIProcessor{
count: 0,
backupService: neDataService.NewBackup,
neInfoService: neService.NewNeInfo,
perfKPIService: neDataService.NewPerfKPI,
count: 0,
backupService: neDataService.NewBackup,
neInfoService: neService.NewNeInfo,
perfKPIService: neDataService.NewPerfKPI,
kpiCReportService: neDataService.NewKpiCReport,
}
// BackupExportKPI 队列任务处理
type BackupExportKPIProcessor struct {
count int // 执行次数
backupService *neDataService.Backup // 备份相关服务
neInfoService *neService.NeInfo // 网元信息服务
perfKPIService *neDataService.PerfKPI // 统计信息服务
count int // 执行次数
backupService *neDataService.Backup // 备份相关服务
neInfoService *neService.NeInfo // 网元信息服务
perfKPIService *neDataService.PerfKPI // 统计信息服务
kpiCReportService *neDataService.KpiCReport // KPI统计服务
}
func (s *BackupExportKPIProcessor) Execute(data any) (any, error) {
@@ -47,7 +49,7 @@ func (s *BackupExportKPIProcessor) Execute(data any) (any, error) {
}
var params struct {
DataType []string `json:"dataType"` // 类型支持 ims/smsc/smf/sgwc
DataType []string `json:"dataType"` // 类型支持 ims/amf/udm/smf/pcf/upf/mme/smsc
FileType string `json:"fileType"` // 文件类型 csv/xlsx
Hour int `json:"hour"` // 数据时间从任务执行时间前的小时数
}
@@ -70,6 +72,8 @@ func (s *BackupExportKPIProcessor) Execute(data any) (any, error) {
if ne.NeType == "UPF" {
interval = 5
}
//
query := neDataModel.GoldKPIQuery{
NeType: ne.NeType,
NeID: ne.NeId,
@@ -78,7 +82,18 @@ func (s *BackupExportKPIProcessor) Execute(data any) (any, error) {
StartTime: fmt.Sprintf("%d", start.UnixMilli()),
EndTime: fmt.Sprintf("%d", end.UnixMilli()),
}
result[ne.NeName] = s.exportKPI(query, params.FileType)
result[ne.NeName+"_kpi"] = s.exportKPI(query, params.FileType)
//
queryC := neDataModel.KPICQuery{
NeType: ne.NeType,
NeID: ne.NeId,
RmUID: ne.RmUID,
Interval: interval,
BeginTime: start.UnixMilli(),
EndTime: end.UnixMilli(),
}
result[ne.NeName+"_kpic"] = s.exportKPIC(queryC, params.FileType)
}
}
@@ -242,3 +257,160 @@ func (s *BackupExportKPIProcessor) exportKPI(query neDataModel.GoldKPIQuery, fil
}
return "ok"
}
// exportKPI 导出KPIC数据
func (s *BackupExportKPIProcessor) exportKPIC(query neDataModel.KPICQuery, fileType string) string {
rows := s.kpiCReportService.FindData(query)
if len(rows) == 0 {
return "no data"
}
// 获取数据指标id
kpiTitleMap := make(map[string]string, 0)
kpiTitles := s.kpiCReportService.FindTitle(query.NeType)
for _, v := range kpiTitles {
kpiTitleMap[v.KpiId] = v.Title
}
// 导出文件名称
dateStr := date.ParseDateToStr(parse.Number(query.EndTime), date.YYYYMMDDHHMMSS)
fileName := fmt.Sprintf("%s_%s_kpic_export_%d_%s.%s", strings.ToLower(query.NeType), query.NeID, len(rows), dateStr, fileType)
filePath := filepath.Join(s.backupService.BACKUP_DIR, fmt.Sprintf("/%s_kpi", strings.ToLower(query.NeType)), fileName)
if runtime.GOOS == "windows" {
filePath = fmt.Sprintf("C:%s", filePath)
}
if fileType == "csv" {
data := [][]string{}
// 获取kpiIdMap的键并排序
var sortedKpiIds []string
for kpiId := range kpiTitleMap {
sortedKpiIds = append(sortedKpiIds, kpiId)
}
sort.Strings(sortedKpiIds) // 按字母排序
// 头 - 按排序后的顺序添加标题
header := []string{}
for _, kpiId := range sortedKpiIds {
header = append(header, kpiTitleMap[kpiId])
}
header = append(header, "NE Name")
header = append(header, "Time")
data = append(data, header)
for _, row := range rows {
// 取时间
timeStr := ""
if v, ok := row["timeGroup"]; ok && v != nil {
if releaseTime := parse.Number(v); releaseTime > 0 {
timeStr = date.ParseDateToStr(releaseTime, date.YYYY_MM_DD_HH_MM_SS_GMT_TZ)
} else {
timeStr = fmt.Sprintf("%s", v)
}
}
// 取网元名称
neName := ""
if v, ok := row["neName"]; ok && v != nil {
neName = fmt.Sprintf("%s", v)
}
// 按排序后的顺序获取数据
dataRow := []string{}
for _, kpiId := range sortedKpiIds {
value := ""
if v, ok := row[kpiId]; ok && v != nil {
value = fmt.Sprint(v)
}
dataRow = append(dataRow, value)
}
dataRow = append(dataRow, neName)
dataRow = append(dataRow, timeStr)
data = append(data, dataRow)
}
// 输出到文件
if err := file.WriterFileCSV(data, filePath); err != nil {
logger.Errorf("export operate log err => %v", err.Error())
return "export err"
}
}
if fileType == "xlsx" {
// 获取kpiIdMap的键并排序
var sortedKpiIds []string
for kpiId := range kpiTitleMap {
sortedKpiIds = append(sortedKpiIds, kpiId)
}
sort.Strings(sortedKpiIds) // 按字母排序
// 第一行表头标题 - 按排序后的顺序
headerCells := map[string]string{}
colIndex := 0
// 先添加KPI标题列
for i, kpiId := range sortedKpiIds {
colLetter := string(rune('A' + i))
headerCells[colLetter+"1"] = kpiTitleMap[kpiId]
colIndex++
}
// 添加NE Name和Time列
neNameCol := string(rune('A' + colIndex))
headerCells[neNameCol+"1"] = "NE Name"
colIndex++
timeCol := string(rune('A' + colIndex))
headerCells[timeCol+"1"] = "Time"
// 从第二行开始的数据
dataCells := make([]map[string]any, 0)
for i, row := range rows {
idx := strconv.Itoa(i + 2)
// 取时间和网元名称
timeStr := ""
if v, ok := row["timeGroup"]; ok && v != nil {
if releaseTime := parse.Number(v); releaseTime > 0 {
timeStr = date.ParseDateToStr(releaseTime, date.YYYY_MM_DD_HH_MM_SS_GMT_TZ)
} else {
timeStr = fmt.Sprintf("%s", v)
}
}
neName := ""
if v, ok := row["neName"]; ok && v != nil {
neName = fmt.Sprintf("%s", v)
}
// 按排序后的顺序填充数据
dataCell := map[string]any{}
// 填充KPI数据
for j, kpiId := range sortedKpiIds {
value := ""
if v, ok := row[kpiId]; ok && v != nil {
value = fmt.Sprint(v)
}
colLetter := string(rune('A' + j))
dataCell[colLetter+idx] = value
}
// 填充NE Name和Time
dataCell[neNameCol+idx] = neName
dataCell[timeCol+idx] = timeStr
dataCells = append(dataCells, dataCell)
}
// 导出数据表格
if err := file.WriterFileExecl(headerCells, dataCells, filePath, ""); err != nil {
logger.Errorf("export operate log err => %v", err.Error())
return "export err"
}
}
// 上传到FTP服务器
if err := s.backupService.FTPPushFile(filePath, ""); err != nil {
return "ok, ftp err:" + err.Error()
}
return "ok"
}

View File

@@ -0,0 +1,52 @@
package model
// KpiCTitle 自定义指标标题信息对象 kpi_title
type KpiCTitle struct {
ID int64 `json:"id" gorm:"column:id;primaryKey;autoIncrement"`
NeType string `json:"neType" gorm:"column:ne_type"`
KpiId string `json:"kpiId" gorm:"column:kpi_id"`
Title string `json:"title" gorm:"column:title"`
Expression string `json:"expression" gorm:"column:expression"`
Unit string `json:"unit" gorm:"column:unit"`
Status string `json:"status" gorm:"column:status"` // 0-Inactive/1-Active/2-Deleted
Description string `json:"description" gorm:"column:description"`
CreatedBy string `json:"createdBy" gorm:"column:created_by"`
UpdatedAt int64 `json:"updatedAt" gorm:"column:updated_at"`
}
// TableName 表名称
func (*KpiCTitle) TableName() string {
return "kpi_c_title"
}
// KpiCReport 自定义指标报表信息对象
type KpiCReport struct {
ID int64 `json:"id" gorm:"column:id;primaryKey;autoIncrement"`
NeType string `json:"neType" gorm:"column:ne_type"`
NeName string `json:"neName" gorm:"column:ne_name"`
RmUid string `json:"rmUid" gorm:"column:rm_uid"`
Date string `json:"date" gorm:"column:date"` // Date of the report yyyy-mm-dd hh:mi:ss
StartTime string `json:"startTime" gorm:"column:start_time"` // Start time of the report hh:mi:ss
EndTime string `json:"endTime" gorm:"column:end_time"` // End time of the report hh:mi:ss
Index int64 `json:"index" gorm:"column:index"` // Index of the report
Granularity int64 `json:"granularity" gorm:"column:granularity"` // Time granualarity: 5/10/.../60/300 (second)
KpiValues string `json:"kpiValues" gorm:"column:kpi_values"` // KPI values JSON String
CreatedAt int64 `json:"createdAt" gorm:"column:created_at"` // Creation time
}
// TableName 表名称
func (*KpiCReport) TableName() string {
return "kpi_c_report"
}
// KPICQuery 指标查询参数结构体
type KPICQuery struct {
NeType string `form:"neType" binding:"required"`
NeID string `form:"neId" binding:"required"`
BeginTime int64 `form:"beginTime" binding:"required"` // 开始时间戳毫秒1739361200999
EndTime int64 `form:"endTime" binding:"required"` // 结束时间戳毫秒1739361210088
Interval int64 `form:"interval" binding:"required,oneof=5 60 300 900 1800 3600"`
RmUID string `form:"rmUID"`
SortField string `form:"sortField" binding:"omitempty,oneof=timeGroup"`
SortOrder string `form:"sortOrder" binding:"omitempty,oneof=asc desc"`
}

View File

@@ -0,0 +1,252 @@
package repository
import (
"fmt"
"sort"
"strings"
"time"
"be.ems/src/framework/database/db"
"be.ems/src/framework/logger"
"be.ems/src/modules/network_data/model"
)
// 实例化数据层 KpiCReport 结构体
var NewKpiCReport = &KpiCReport{}
// KpiCReport 性能统计 数据层处理
type KpiCReport struct{}
// SelectGoldKPI 通过网元指标数据信息
func (r KpiCReport) SelectKPI(query model.KPICQuery) []model.KpiCReport {
rows := []model.KpiCReport{}
if query.NeType == "" {
return rows
}
tx := db.DB("").Model(&model.KpiCReport{})
// 表名
tableName := fmt.Sprintf("kpi_c_report_%s", strings.ToLower(query.NeType))
tx.Table(tableName)
// 构建查询条件
if query.RmUID != "" {
tx = tx.Where("rm_uid = ?", query.RmUID)
}
if query.BeginTime != 0 {
tx = tx.Where("created_at >= ?", query.BeginTime)
}
if query.EndTime != 0 {
tx = tx.Where("created_at <= ?", query.EndTime)
}
// 查询数据
if err := tx.Find(&rows).Error; err != nil {
logger.Errorf("query find err => %v", err.Error())
return rows
}
// 排序
if query.SortField != "" {
sortField := query.SortField
sortOrder := query.SortOrder
sort.SliceStable(rows, func(i, j int) bool {
// 支持的排序字段映射
fieldGetters := map[string]func(*model.KpiCReport) any{
"id": func(row *model.KpiCReport) any { return row.ID },
"timeGroup": func(row *model.KpiCReport) any { return row.CreatedAt },
"createdAt": func(row *model.KpiCReport) any { return row.CreatedAt },
// 可添加更多支持的字段
}
// 获取字段 getter 函数
getter, ok := fieldGetters[sortField]
if !ok {
// 非法字段使用默认排序id升序
return rows[i].ID < rows[j].ID
}
// 获取比较值
valI, valJ := getter(&rows[i]), getter(&rows[j])
// 根据字段类型进行比较
switch v := valI.(type) {
case int64:
if sortOrder == "desc" {
return v > valJ.(int64)
}
return v < valJ.(int64)
case string:
if sortOrder == "desc" {
return v > valJ.(string)
}
return v < valJ.(string)
default:
// 不支持的字段类型,使用默认排序
return rows[i].ID < rows[j].ID
}
})
}
return rows
}
// Insert 新增信息 返回新增数据ID
func (r KpiCReport) Insert(param model.KpiCReport) int64 {
if param.NeType == "" {
return 0
}
if param.CreatedAt == 0 {
param.CreatedAt = time.Now().UnixMilli()
}
// 表名
tableName := fmt.Sprintf("kpi_c_report_%s", strings.ToLower(param.NeType))
// 执行插入
if err := db.DB("").Table(tableName).Create(&param).Error; err != nil {
logger.Errorf("insert err => %v", err.Error())
return 0
}
return param.ID
}
// SelectKPITitle 网元对应的指标名称
func (r KpiCReport) SelectKPITitle(neType string) []model.KpiCTitle {
rows := []model.KpiCTitle{}
if neType == "" {
return rows
}
tx := db.DB("").Model(&model.KpiCTitle{})
// 构建查询条件
tx = tx.Where("ne_type =?", neType)
// 查询数据
if err := tx.Find(&rows).Error; err != nil {
logger.Errorf("query find err => %v", err.Error())
return rows
}
return rows
}
// TitleLastKPIId 查询指标标题最后kpiid
func (r KpiCReport) TitleLastKPIId(neType string) string {
tx := db.DB("").Model(&model.KpiCTitle{})
tx = tx.Where("ne_type=?", neType)
tx = tx.Select("kpi_id").Order("kpi_id DESC")
// 查询数据
var kpiId string = ""
if err := tx.Limit(1).Find(&kpiId).Error; err != nil {
logger.Errorf("query find err => %v", err.Error())
return kpiId
}
return kpiId
}
// SelectByPageTitle 分页查询集合
func (r KpiCReport) TitleSelectByPage(query map[string]string) ([]model.KpiCTitle, int64) {
tx := db.DB("").Model(&model.KpiCTitle{})
// 查询条件拼接
if v, ok := query["neType"]; ok && v != "" {
tx = tx.Where("ne_type = ?", v)
}
if v, ok := query["status"]; ok && v != "" {
tx = tx.Where("status = ?", v)
} else {
tx = tx.Where("status != ?", "2")
}
// 查询结果
var total int64 = 0
rows := []model.KpiCTitle{}
// 查询数量 长度为0直接返回
if err := tx.Count(&total).Error; err != nil || total <= 0 {
return rows, total
}
// 分页
pageNum, pageSize := db.PageNumSize(query["pageNum"], query["pageSize"])
tx = tx.Offset(int(pageNum * pageSize)).Limit(int(pageSize))
// 排序
if v, ok := query["sortField"]; ok && v != "" {
sortSql := v
if o, ok := query["sortOrder"]; ok && o != "" {
if o == "desc" {
sortSql += " desc "
} else {
sortSql += " asc "
}
}
tx = tx.Order(sortSql)
}
// 查询数据
if err := tx.Find(&rows).Error; err != nil {
logger.Errorf("query err => %v", err)
}
return rows, total
}
// TitleSelect 网元对应的指标名称
func (r KpiCReport) TitleSelect(param model.KpiCTitle) []model.KpiCTitle {
tx := db.DB("").Model(&model.KpiCTitle{})
// 构建查询条件
if param.NeType != "" {
tx = tx.Where("ne_type =?", param.NeType)
}
if param.Title != "" {
tx = tx.Where("title = ?", param.Title)
}
if param.Status != "" {
tx = tx.Where("status = ?", param.Status)
}
// 查询数据
rows := []model.KpiCTitle{}
if err := tx.Find(&rows).Error; err != nil {
logger.Errorf("query find err => %v", err.Error())
return rows
}
return rows
}
// TitleInsert 新增信息
func (r KpiCReport) TitleInsert(param model.KpiCTitle) int64 {
if param.CreatedBy != "" {
param.UpdatedAt = time.Now().UnixMilli()
}
param.Status = "1"
tx := db.DB("").Create(&param)
if err := tx.Error; err != nil {
logger.Errorf("CreateInBatches err => %v", err)
}
return param.ID
}
// TitleUpdate 修改信息
func (r KpiCReport) TitleUpdate(param model.KpiCTitle) int64 {
if param.ID <= 0 {
return 0
}
param.UpdatedAt = time.Now().UnixMilli()
tx := db.DB("").Model(&model.KpiCTitle{})
// 构建查询条件
tx = tx.Where("id = ?", param.ID)
tx = tx.Omit("id", "created_by")
// 执行更新
if err := tx.Updates(param).Error; err != nil {
logger.Errorf("update err => %v", err.Error())
return 0
}
return tx.RowsAffected
}
// TitleDeleteByIds 批量删除信息
func (r KpiCReport) TitleDeleteByIds(ids []int64) int64 {
if len(ids) <= 0 {
return 0
}
tx := db.DB("").Model(&model.KpiCTitle{})
// 构建查询条件
tx = tx.Where("id in ?", ids)
if err := tx.Update("status", 2).Error; err != nil {
logger.Errorf("update err => %v", err.Error())
return 0
}
return tx.RowsAffected
}

View File

@@ -0,0 +1,192 @@
package service
import (
"encoding/json"
"fmt"
"sort"
"be.ems/src/framework/utils/parse"
"be.ems/src/modules/network_data/model"
"be.ems/src/modules/network_data/repository"
)
// 实例化数据层 KpiCReport 结构体
var NewKpiCReport = &KpiCReport{
kpiCReportRepository: repository.NewKpiCReport,
}
// KpiCReport 自定义性能统计 服务层处理
type KpiCReport struct {
kpiCReportRepository *repository.KpiCReport // 自定义KPI数据信息
}
// FindKPI 通过网元指标数据信息
func (s KpiCReport) FindData(query model.KPICQuery) []map[string]any {
// 标题单位映射
kpicTitles := s.kpiCReportRepository.SelectKPITitle(query.NeType)
kpicTitleUnitMap := map[string]string{}
for _, v := range kpicTitles {
kpicTitleUnitMap[v.KpiId] = v.Unit
}
// 原始数据
rows := s.kpiCReportRepository.SelectKPI(query)
if len(rows) <= 0 {
return []map[string]any{}
}
kpiIdsHas := false
kpiIds := []string{}
// 处理数据
arr := []map[string]any{}
for _, row := range rows {
// 解析 JSON 字符串为 map
var kpiValues []map[string]any
err := json.Unmarshal([]byte(row.KpiValues), &kpiValues)
if err != nil {
continue
}
item := map[string]any{
"neType": row.NeType,
"neName": row.NeName,
"rmUID": row.RmUid,
"startIndex": row.Index,
"timeGroup": row.CreatedAt,
}
// 遍历 kpiValues 数组
for _, v := range kpiValues {
kpiId := "-"
if k, ok := v["kpiId"]; ok {
kpiId = fmt.Sprint(k)
}
if k, ok := v["kpi_id"]; ok {
kpiId = fmt.Sprint(k)
}
item[kpiId] = v["value"]
}
arr = append(arr, item)
// 添加指标ID
if !kpiIdsHas {
for _, v := range kpiValues {
kpiId := "-"
if k, ok := v["kpiId"]; ok {
kpiId = fmt.Sprint(k)
}
if k, ok := v["kpi_id"]; ok {
kpiId = fmt.Sprint(k)
}
kpiIds = append(kpiIds, kpiId)
}
kpiIdsHas = true
}
}
// 时间密度分钟 数值单位秒 5分钟的传入300秒
timeInterval := query.Interval
// 创建一个map来存储按时间段合并后的数据
timeGroup := make(map[int64][]map[string]any)
// 遍历每个数据项
for _, v := range arr {
itemTime := parse.Number(v["timeGroup"])
// 计算时间戳的x分钟时间段使用秒并除以x分钟
timeMinute := itemTime / 1000 / timeInterval * timeInterval
// 合并到对应的时间段
timeGroup[timeMinute] = append(timeGroup[timeMinute], v)
}
// 时间组合输出
data := []map[string]any{}
for _, records := range timeGroup {
if len(records) <= 0 {
continue
}
// 转换为具体时间显示(根据需要可以格式化显示)
// timeStr := time.Unix(k, 0).Format("2006-01-02 15:04:05")
// fmt.Printf("Time Group: %s records: %d\n", timeStr, len(records))
startItem := records[len(records)-1] // 取最后一条数据也是最开始startIndex
if len(records) >= 2 { // 最后一条数据不参与计算
for _, record := range records[:len(records)-1] {
// fmt.Printf(" - startIndex: %v, Value: %v\n", record["startIndex"], record["timeGroup"])
// 遍历kpiIds数组对lastRecord赋值
for _, kpiId := range kpiIds {
if v, ok := record[kpiId]; ok {
value := v.(float64) + startItem[kpiId].(float64)
startItem[kpiId] = value
// value := parse.Number(startItem[kpiId])
// startItem[kpiId] = value + parse.Number(v)
}
}
}
// 处理单位
for _, kpiId := range kpiIds {
unit, ok := kpicTitleUnitMap[kpiId]
if !ok {
continue
}
// "Mbps" "%"
if unit == "%" {
startItem[kpiId] = startItem[kpiId].(float64) / float64(len(records))
}
}
}
data = append(data, startItem)
}
// 按时间排序
sort.SliceStable(data, func(i, j int) bool {
vi := parse.Number(data[i]["timeGroup"])
vj := parse.Number(data[j]["timeGroup"])
if query.SortOrder == "asc" {
return vi < vj // asc
}
return vi > vj // desc
})
return data
}
// Insert 新增信息
func (s KpiCReport) Insert(param model.KpiCReport) int64 {
return s.kpiCReportRepository.Insert(param)
}
// FindKPITitle 网元对应的指标名称
func (r KpiCReport) FindTitle(neType string) []model.KpiCTitle {
return r.kpiCReportRepository.SelectKPITitle(neType)
}
// TitleLastKPIId 指标标题最后kpiid
func (r KpiCReport) TitleLastKPIId(neType string) string {
return r.kpiCReportRepository.TitleLastKPIId(neType)
}
// FindByPage 根据条件分页查询
func (r KpiCReport) TitleFindByPage(query map[string]string) ([]model.KpiCTitle, int64) {
return r.kpiCReportRepository.TitleSelectByPage(query)
}
// TitleFind 查询信息
func (r KpiCReport) TitleFind(param model.KpiCTitle) []model.KpiCTitle {
return r.kpiCReportRepository.TitleSelect(param)
}
// TitleUpdate 更新信息
func (r KpiCReport) TitleUpdate(param model.KpiCTitle) int64 {
return r.kpiCReportRepository.TitleUpdate(param)
}
// TitleDeleteByIds 批量删除信息
func (r KpiCReport) TitleDeleteByIds(ids []int64) (int64, error) {
rows := r.kpiCReportRepository.TitleDeleteByIds(ids)
if rows > 0 {
return rows, nil
}
// 删除信息失败!
return 0, fmt.Errorf("delete fail")
}
// TitleInsert 新增信息
func (r KpiCReport) TitleInsert(param model.KpiCTitle) int64 {
return r.kpiCReportRepository.TitleInsert(param)
}