From 41a0090dc911e34f1d19d8f0f85b0bb71ae74070 Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Fri, 26 Apr 2024 15:40:17 +0800 Subject: [PATCH] opt: kpi performance optimize --- database/install/kpi_report.sql | 39 ++++++++++++++++ database/upgrade/upg_kpi_report.sql | 38 +++++++++++++++ database/upgvue3/upg_kpi_report.sql | 38 +++++++++++++++ features/pm/performance.go | 71 ++++++++++++++++++++++++++++- misc/importdb.sh | 13 +++++- 5 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 database/install/kpi_report.sql create mode 100644 database/upgrade/upg_kpi_report.sql create mode 100644 database/upgvue3/upg_kpi_report.sql diff --git a/database/install/kpi_report.sql b/database/install/kpi_report.sql new file mode 100644 index 00000000..490046e4 --- /dev/null +++ b/database/install/kpi_report.sql @@ -0,0 +1,39 @@ +/* + Navicat Premium Data Transfer + + Source Server : local_mariadb + Source Server Type : MariaDB + Source Server Version : 100338 (10.3.38-MariaDB) + Source Host : localhost:33066 + Source Schema : omc_db + + Target Server Type : MariaDB + Target Server Version : 100338 (10.3.38-MariaDB) + File Encoding : 65001 + + Date: 26/04/2024 10:33:26 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for kpi_report +-- ---------------------------- +DROP TABLE IF EXISTS `kpi_report`; +CREATE TABLE `kpi_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `ne_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `ne_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '', + `rm_uid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `date` date NOT NULL, + `start_time` time NULL DEFAULT NULL, + `end_time` time NULL DEFAULT NULL, + `index` smallint(6) NOT NULL, + `granularity` tinyint(4) NULL DEFAULT 60 COMMENT 'Time granualarity: 5/10/.../60/300 (second)', + `kpi_values` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL, + `created_at` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 10660 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_kpi_report.sql b/database/upgrade/upg_kpi_report.sql new file mode 100644 index 00000000..924921f3 --- /dev/null +++ b/database/upgrade/upg_kpi_report.sql @@ -0,0 +1,38 @@ +/* + Navicat Premium Data Transfer + + Source Server : local_mariadb + Source Server Type : MariaDB + Source Server Version : 100338 (10.3.38-MariaDB) + Source Host : localhost:33066 + Source Schema : omc_db + + Target Server Type : MariaDB + Target Server Version : 100338 (10.3.38-MariaDB) + File Encoding : 65001 + + Date: 26/04/2024 10:33:46 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for kpi_report +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `kpi_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `ne_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `ne_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '', + `rm_uid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `date` date NOT NULL, + `start_time` time NULL DEFAULT NULL, + `end_time` time NULL DEFAULT NULL, + `index` smallint(6) NOT NULL, + `granularity` tinyint(4) NULL DEFAULT 60 COMMENT 'Time granualarity: 5/10/.../60/300 (second)', + `kpi_values` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL, + `created_at` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 10660 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgvue3/upg_kpi_report.sql b/database/upgvue3/upg_kpi_report.sql new file mode 100644 index 00000000..3ef1fd41 --- /dev/null +++ b/database/upgvue3/upg_kpi_report.sql @@ -0,0 +1,38 @@ +/* + Navicat Premium Data Transfer + + Source Server : local_mariadb + Source Server Type : MariaDB + Source Server Version : 100338 (10.3.38-MariaDB) + Source Host : localhost:33066 + Source Schema : omc_db + + Target Server Type : MariaDB + Target Server Version : 100338 (10.3.38-MariaDB) + File Encoding : 65001 + + Date: 26/04/2024 10:34:01 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for kpi_report +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `kpi_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `ne_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `ne_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '', + `rm_uid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `date` date NOT NULL, + `start_time` time NULL DEFAULT NULL, + `end_time` time NULL DEFAULT NULL, + `index` smallint(6) NOT NULL, + `granularity` tinyint(4) NULL DEFAULT 60 COMMENT 'Time granualarity: 5/10/.../60/300 (second)', + `kpi_values` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL, + `created_at` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 10660 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/features/pm/performance.go b/features/pm/performance.go index 6be6dc64..e8db2f5b 100644 --- a/features/pm/performance.go +++ b/features/pm/performance.go @@ -8,6 +8,7 @@ import ( "math" "net/http" "strconv" + "strings" "time" "be.ems/lib/dborm" @@ -63,6 +64,26 @@ type GoldKpi struct { Timestamp string `json:"timestamp"` } +type KpiData struct { + ID int `json:"id" xorm:"pk 'id' '<-' autoincr"` + NEType string `json:"neType" xorm:"ne_type"` + NEName string `json:"neName" xorm:"ne_name"` + RmUid string `json:"rmUid" xorm:"rm_uid"` + Date string `json:"date" xorm:"date"` + StartTime time.Time `json:"startTime" xorm:"start_time"` + EndTime time.Time `json:"endTime" xorm:"end_time"` + Index int `json:"index" xorm:"index"` + Granularity int8 `json:"granularity" xorm:"granularity"` + KPIValues []KPIVal `json:"kpiValues" xorm:"json 'kpi_values'"` + //CreatedAt int64 `json:"createdAt" xorm:"created 'created_at'"` + CreatedAt int64 `json:"createdAt" xorm:"'created_at'"` +} +type KPIVal struct { + KPIID string `json:"kpiID"` + Value int64 `json:"value"` + Err string `json:"err"` +} + var ( // performance management PerformanceUri = config.DefaultUriPrefix + "/performanceManagement/{apiVersion}/elementType/{elementTypeValue}/objectType/kpiReport/{index}" @@ -124,6 +145,21 @@ func InitDbClient(dbType, dbUser, dbPassword, dbHost, dbPort, dbName, dbParam st } xEngine = DbClient.XEngine + // exist, err := xEngine.IsTableExist("kpi_report") + // if err != nil { + // log.Error("Failed to IsTableExist:", err) + // return err + // } + // if exist { + // // 复制表结构到新表 + // sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` AS SELECT * FROM kpi_report WHERE 1=0", "kpi_report_amf") + // _, err := xEngine.Exec(sql) + // if err != nil { + // log.Error("Failed to Exec:", err) + // return err + // } + // } + return nil } @@ -204,12 +240,34 @@ func PostKPIReportFromNF(w http.ResponseWriter, r *http.Request) { "startIndex": goldKpi.Index, "timeGroup": goldKpi.StartTime, } + // insert into new kpi_report_xxx table + kpiData := new(KpiData) + kpiData.Date = goldKpi.Date + kpiData.Index = goldKpi.Index + st, _ := time.ParseInLocation(time.RFC3339Nano, kpiReport.Task.Period.StartTime, time.Local) + et, _ := time.ParseInLocation(time.RFC3339Nano, kpiReport.Task.Period.EndTime, time.Local) + // kpiData.StartTime = goldKpi.StartTime + // kpiData.EndTime = goldKpi.EndTime + kpiData.StartTime = st + kpiData.EndTime = et + kpiData.Granularity = goldKpi.Granularity + kpiData.NEName = goldKpi.NEName + kpiData.NEType = goldKpi.NEType + kpiData.RmUid = goldKpi.RmUid + kpiVal := new(KPIVal) + kpiData.CreatedAt = time.Now().UnixMilli() for _, k := range kpiReport.Task.NE.KPIs { kpiEvent[k.KPIID] = k.Value // kip_id goldKpi.KpiId = k.KPIID goldKpi.Value = k.Value goldKpi.Error = k.Err - log.Trace("goldKpi:", goldKpi) + + kpiVal.KPIID = k.KPIID + kpiVal.Value = int64(k.Value) + kpiVal.Err = k.Err + kpiData.KPIValues = append(kpiData.KPIValues, *kpiVal) + + //log.Trace("goldKpi:", goldKpi) // 启动事务 err := session.Begin() @@ -243,13 +301,22 @@ func PostKPIReportFromNF(w http.ResponseWriter, r *http.Request) { } } + // insert kpi_report table, no session + tableName := "kpi_report_" + strings.ToLower(kpiReport.Task.NE.NeType) + affected, err := xEngine.Table(tableName).Insert(kpiData) + if err != nil && affected <= 0 { + log.Errorf("Failed to insert %s:%v", tableName, err) + services.ResponseInternalServerError500ProcessError(w, err) + return + } + // 推送到ws订阅组 wsService.NewWSSendImpl.ByGroupID(wsService.GROUP_KPI, kpiEvent) if goldKpi.NEType == "UPF" { wsService.NewWSSendImpl.ByGroupID(wsService.GROUP_KPI_UPF, kpiEvent) } - services.ResponseStatusOK200Null(w) + services.ResponseStatusOK204NoContent(w) } type MeasureTask struct { diff --git a/misc/importdb.sh b/misc/importdb.sh index 9a3d7dfa..11113f0a 100644 --- a/misc/importdb.sh +++ b/misc/importdb.sh @@ -18,7 +18,6 @@ case "$1" in echo "Execute SQL script: ${SQL} ..." mysql -u${USER} -p${PASSWORD} -P ${PORT} --protocol tcp -D ${DBNAME} < ${SQL}; done - ;; upgvue3) @@ -97,3 +96,15 @@ case "$1" in done ;; esac + +# create kpi_report table with ne_type, exp: kpi_report_amf +ne_types=$(mysql -u${USER} -p${PASSWORD} -P ${PORT} --protocol tcp -D ${DBNAME} -se "SELECT DISTINCT LOWER(ne_type) FROM kpi_title") +for ne_type in ${ne_types}; do + TABLE_NAME="kpi_report_${ne_type}" + SQL="CREATE TABLE IF NOT EXISTS ${TABLE_NAME} AS SELECT * FROM kpi_report WHERE 1=0" + echo -n "Create ${TABLE_NAME} table ..." + mysql -u${USER} -p${PASSWORD} -P ${PORT} --protocol tcp -D ${DBNAME} -e "${SQL}"; + if [ $? = 0 ]; then + echo "done" + fi +done \ No newline at end of file