From 13b322a038fe88445caf731c32862e6bdf83d04d Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Fri, 6 Sep 2024 14:58:02 +0800 Subject: [PATCH] add: update export table task --- database/install/sys_job.sql | 2 +- database/upgrade/upg_sys_job.sql | 2 +- lib/dborm/dbgorm.go | 8 +++++++- lib/dborm/dborm.go | 5 +++++ src/modules/crontask/processor/exportTable/exportTable.go | 4 ++-- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/database/install/sys_job.sql b/database/install/sys_job.sql index 131ad511..938765e1 100644 --- a/database/install/sys_job.sql +++ b/database/install/sys_job.sql @@ -37,7 +37,7 @@ INSERT INTO `sys_job` VALUES (7, 'job.backupEtcFromNE', 'SYSTEM', 'backupEtcFrom INSERT INTO `sys_job` VALUES (8, 'job.deleteExpiredNeStateRecord', 'SYSTEM', 'deleteExpiredRecord', '{\"duration\":1,\"tableName\":\"ne_state\",\"colName\":\"timestamp\"}', '0 25 0 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1703668901929, 'job.deleteExpiredNeStateRecordRemark'); INSERT INTO `sys_job` VALUES (9, 'job.getStateFromNE', 'SYSTEM', 'getStateFromNE', '', '0/10 * * * * ?', '3', '0', '0', '0', 'supervisor', 1698478134842, 'admin', 1713231120503, 'job.getStateFromNERemark'); INSERT INTO `sys_job` VALUES (10, 'job.genNeStateAlarm', 'SYSTEM', 'genNeStateAlarm', '{\"alarmID\":\"HXEMSSM10000\",\"alarmCode\":10000,\"alarmTitle\":\"The system state is abnormal\",\"neType\":\"OMC\",\"alarmType\":\"EquipmentAlarm\",\"origSeverity\": \"Major\",\"objectName\":\"EMS;SystemManagement;Heartbeat\",\"objectType\":\"SystemState\",\"specificProblem\":\"Alarm cause: the system state of target NE has not been received for {threshold} seconds\", \"specificProblemID\":\"AC10000\",\"threshold\":30}', '0/5 * * * * ?', '3', '0', '0', '0', 'supervisor', 1698478134842, 'admin', 1713781643031, 'job.genNeStateAlarmRemark'); -INSERT INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time,tenant_id\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); +INSERT INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); INSERT INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); INSERT INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,ne_type,ne_name,rm_uid,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); diff --git a/database/upgrade/upg_sys_job.sql b/database/upgrade/upg_sys_job.sql index 48332084..00b91c64 100644 --- a/database/upgrade/upg_sys_job.sql +++ b/database/upgrade/upg_sys_job.sql @@ -41,7 +41,7 @@ REPLACE INTO `sys_job` VALUES (7, 'job.backupEtcFromNE', 'SYSTEM', 'backupEtcFro REPLACE INTO `sys_job` VALUES (8, 'job.deleteExpiredNeStateRecord', 'SYSTEM', 'deleteExpiredRecord', '{\"duration\":1,\"tableName\":\"ne_state\",\"colName\":\"timestamp\"}', '0 25 0 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1703668901929, 'job.deleteExpiredNeStateRecordRemark'); REPLACE INTO `sys_job` VALUES (9, 'job.getStateFromNE', 'SYSTEM', 'getStateFromNE', '', '0/10 * * * * ?', '3', '0', '0', '0', 'supervisor', 1698478134842, 'admin', 1713231120503, 'job.getStateFromNERemark'); REPLACE INTO `sys_job` VALUES (10, 'job.genNeStateAlarm', 'SYSTEM', 'genNeStateAlarm', '{\"alarmID\":\"HXEMSSM10000\",\"alarmCode\":10000,\"alarmTitle\":\"The system state is abnormal\",\"neType\":\"OMC\",\"alarmType\":\"EquipmentAlarm\",\"origSeverity\": \"Major\",\"objectName\":\"EMS;SystemManagement;Heartbeat\",\"objectType\":\"SystemState\",\"specificProblem\":\"Alarm cause: the system state of target NE has not been received for {threshold} seconds\", \"specificProblemID\":\"AC10000\",\"threshold\":30}', '0/5 * * * * ?', '3', '0', '0', '0', 'supervisor', 1698478134842, 'admin', 1713781643031, 'job.genNeStateAlarmRemark'); -REPLACE INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time,tenant_id\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); +REPLACE INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); REPLACE INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); REPLACE INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,ne_type,ne_name,rm_uid,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); diff --git a/lib/dborm/dbgorm.go b/lib/dborm/dbgorm.go index 95f794b8..30ff7c62 100644 --- a/lib/dborm/dbgorm.go +++ b/lib/dborm/dbgorm.go @@ -1,6 +1,7 @@ package dborm import ( + "database/sql" "fmt" "log" "os" @@ -85,11 +86,16 @@ func Close() { } } -// 获取默认数据源 +// default gorm DB func DefaultDB() *gorm.DB { return dbgEngine } +// get sql DB +func GCoreDB() (*sql.DB, error) { + return dbgEngine.DB() +} + // RawSQL 原生查询语句 func RawSQL(sql string, parameters []any) ([]map[string]any, error) { // 数据源 diff --git a/lib/dborm/dborm.go b/lib/dborm/dborm.go index aa008291..22be07ba 100644 --- a/lib/dborm/dborm.go +++ b/lib/dborm/dborm.go @@ -16,6 +16,7 @@ import ( _ "github.com/go-sql-driver/mysql" "xorm.io/xorm" + "xorm.io/xorm/core" ) const ( @@ -124,6 +125,10 @@ func XormConnectDatabase(dbType, dbUser, dbPassword, dbHost, dbPort, dbName stri return xEngine, nil } +func XCoreDB() *core.DB { + return xEngine.DB() +} + func ConstructInsertSQL(tableName string, insertData interface{}) (string, []string) { log.Debug("ConstructInsertSQL processing... ") log.Debug("Request insertData:", insertData) diff --git a/src/modules/crontask/processor/exportTable/exportTable.go b/src/modules/crontask/processor/exportTable/exportTable.go index 8411f7ce..e0f8a986 100644 --- a/src/modules/crontask/processor/exportTable/exportTable.go +++ b/src/modules/crontask/processor/exportTable/exportTable.go @@ -8,7 +8,7 @@ import ( "os" "time" - "be.ems/lib/core/datasource" + "be.ems/lib/dborm" "be.ems/lib/log" "be.ems/src/framework/cron" ) @@ -103,7 +103,7 @@ func (s *BarProcessor) Execute(data any) (any, error) { } func (s *BarProcessor) exportData(query, filePath string) (int64, error) { - rows, err := datasource.DefaultDB().DB().Query(query) + rows, err := dborm.XCoreDB().Query(query) if err != nil { return 0, err }