From 5661b7c2020dd16eacaa903477a923bafc210e01 Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Wed, 9 Oct 2024 11:33:56 +0800 Subject: [PATCH 1/5] update sshsvc yaml default setting --- config/etc/default/sshsvc.yaml | 16 ++++++++-------- sshsvc/setHLRServiceState | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/config/etc/default/sshsvc.yaml b/config/etc/default/sshsvc.yaml index 3fc51b42..414c397e 100644 --- a/config/etc/default/sshsvc.yaml +++ b/config/etc/default/sshsvc.yaml @@ -32,7 +32,7 @@ sshd: userName: manager password: pass123 authType: local - tagNE: hlr + tagNE: omc # authType: local/omc telnetServer: @@ -45,7 +45,7 @@ telnetServer: userName: manager password: pass123 authType: local - tagNE: hlr + tagNE: omc # authproto: NoAuth/MD5/SHA # privProto: NoPriv/DES/AES/AES192/AES256 @@ -63,15 +63,15 @@ snmpServer: trapBool: false trapTick: 60 timeOut: 5 - trapTarget: "2001:db8::9219" + trapTarget: "" database: type: mysql - user: administrator - password: "*86#ROtartsinim" - host: 192.168.13.117 - port: 3306 - name: OMC_PUB + user: root + password: 1000omc@kp! + host: 127.0.0.1 + port: 33066 + name: omc_db connParam: charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&interpolateParams=True omc: diff --git a/sshsvc/setHLRServiceState b/sshsvc/setHLRServiceState index c792036b..dee6e381 100644 --- a/sshsvc/setHLRServiceState +++ b/sshsvc/setHLRServiceState @@ -2,21 +2,22 @@ TargetIP="[2001:db8::9166]" PORT="34957" +OID=".1.3.6.1.4.1.1379.2.3.3.3.1.1.9.0" case "$1" in 1) echo -n "Set HLR state link down ... " - snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} .1.3.6.1.4.1.1379.2.3.3.3.1.1.9.0 i 1 >/dev/null + snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} ${OID} i 1 >/dev/null echo "done" ;; 2) echo -n "Set HLR state link up ... " - snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} .1.3.6.1.4.1.1379.2.3.3.3.1.1.9.0 i 2 >/dev/null + snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} ${OID} i 2 >/dev/null echo "done" ;; 3) echo -n "Set HLR state authentication failure ... " - snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} .1.3.6.1.4.1.1379.2.3.3.3.1.1.9.0 i 3 >/dev/null + snmpset -v3 -l noAuthNoPriv -u manager ${TargetIP}:${PORT} ${OID} i 3 >/dev/null echo "done" ;; *) From affd4ef15aaa177a67bf47993ab33ed85a3018b3 Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Thu, 10 Oct 2024 10:02:58 +0800 Subject: [PATCH 2/5] update lst neinfo OMC MML status alias message --- database/common/mml_http_map.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/common/mml_http_map.sql b/database/common/mml_http_map.sql index ea729c9e..48d9e647 100644 --- a/database/common/mml_http_map.sql +++ b/database/common/mml_http_map.sql @@ -42,7 +42,7 @@ CREATE TABLE `mml_http_map` ( -- ---------------------------- INSERT INTO `mml_http_map` VALUES (1, 'OMC', 'dsp', 'sysinfo', 'Get', '/api/rest/systemManagement/v1/sysInfo', '/%s', NULL, NULL, '{}', '{\r\n \"retFmt\": \"GetNF\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"title\": \"NE System Information\",\r\n \"singleList\": true,\r\n \"sepSpaceNum\": 2,\r\n \"alignmentM\": \"Left\",\r\n \"alignmentSN\": \"Right\",\r\n \"alignmentSV\": \"Left\",\r\n \"cols\": [\r\n {\r\n \"name\": \"neType\",\r\n \"display\": \"NE type\",\r\n \"length\": 10\r\n },\r\n {\r\n \"name\": \"neId\",\r\n \"display\": \"NE ID\",\r\n \"length\": 28\r\n }, \r\n {\r\n \"name\": \"hostName\",\r\n \"display\": \"Host name\",\r\n \"length\": 16\r\n },\r\n {\r\n \"name\": \"osInfo\",\r\n \"display\": \"OS information\",\r\n \"length\": 128\r\n },\r\n {\r\n \"name\": \"dbInfo\",\r\n \"display\": \"Database information\",\r\n \"length\": 80\r\n },\r\n {\r\n \"name\": \"version\",\r\n \"display\": \"Software version\",\r\n \"length\": 12\r\n },\r\n {\r\n \"name\": \"ipAddr\",\r\n \"display\": \"IP address\",\r\n \"length\": 20\r\n },\r\n {\r\n \"name\": \"port\",\r\n \"display\": \"Port\",\r\n \"length\": 6\r\n },\r\n {\r\n \"name\": \"cpus\",\r\n \"display\": \"CPUs\",\r\n \"length\": 4\r\n },\r\n {\r\n \"name\": \"totalMem\",\r\n \"display\": \"Total memory(KB)\",\r\n \"length\": 11\r\n } \r\n ],\r\n \"end\": \"(Number of results = %d)\\n\\n\"\r\n}'); INSERT INTO `mml_http_map` VALUES (2, 'OMC', 'lst', 'memap', 'Get', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '?loc=', '{}', '{\r\n \"retFmt\": \"GetDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"title\": \"Managed Element Map\",\r\n \"singleList\": true,\r\n \"sepSpaceNum\": 2,\r\n \"alignmentM\": \"Left\",\r\n \"alignmentSN\": \"Right\",\r\n \"alignmentSV\": \"Left\",\r\n \"cols\": [\r\n {\r\n \"name\": \"rm_uid\",\r\n \"display\": \"Resource management UID\",\r\n \"length\": 28\r\n },\r\n {\r\n \"name\": \"ne_name\",\r\n \"display\": \"NE name\",\r\n \"length\": 28\r\n }\r\n ],\r\n \"end\": \"(Number of results = %d)\\n\\n\"\r\n}'); -INSERT INTO `mml_http_map` VALUES (3, 'OMC', 'lst', 'neinfo', 'Get', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '?loc=', '{}', '{\r\n \"retFmt\": \"GetDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"title\": \"Network element information\",\r\n \"singleList\": true,\r\n \"sepSpaceNum\": 2,\r\n \"alignmentM\": \"Left\",\r\n \"alignmentSN\": \"Right\",\r\n \"alignmentSV\": \"Left\",\r\n \"cols\": [\r\n {\r\n \"name\": \"ne_type\",\r\n \"display\": \"NE type\",\r\n \"length\": 10\r\n },\r\n {\r\n \"name\": \"ne_id\",\r\n \"display\": \"NE ID\",\r\n \"length\": 16\r\n },\r\n {\r\n \"name\": \"rm_uid\",\r\n \"display\": \"Resource management UID\",\r\n \"length\": 28\r\n },\r\n {\r\n \"name\": \"ne_name\",\r\n \"display\": \"NE name\",\r\n \"length\": 28\r\n },\r\n {\r\n \"name\": \"pv_flag\",\r\n \"display\": \"PV flag\",\r\n \"length\": 10\r\n },\r\n {\r\n \"name\": \"ip\",\r\n \"display\": \"IP address\",\r\n \"length\": 32\r\n },\r\n {\r\n \"name\": \"port\",\r\n \"display\": \"Port\",\r\n \"length\": 6\r\n },\r\n {\r\n \"name\": \"status\",\r\n \"display\": \"Status\",\r\n \"length\": 10,\r\n \"alias\": [\r\n \"online\",\r\n \"offline\",\r\n \"standby\",\r\n \"maintain\"\r\n ]\r\n }\r\n ],\r\n \"end\": \"(Number of results = %d)\\n\\n\"\r\n}\r\n'); +INSERT INTO `mml_http_map` VALUES (3, 'OMC', 'lst', 'neinfo', 'Get', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '?loc=', '{}', '{\r\n \"retFmt\": \"GetDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"title\": \"Network element information\",\r\n \"singleList\": true,\r\n \"sepSpaceNum\": 2,\r\n \"alignmentM\": \"Left\",\r\n \"alignmentSN\": \"Right\",\r\n \"alignmentSV\": \"Left\",\r\n \"cols\": [\r\n {\r\n \"name\": \"ne_type\",\r\n \"display\": \"NE type\",\r\n \"length\": 10\r\n },\r\n {\r\n \"name\": \"ne_id\",\r\n \"display\": \"NE ID\",\r\n \"length\": 16\r\n },\r\n {\r\n \"name\": \"rm_uid\",\r\n \"display\": \"Resource management UID\",\r\n \"length\": 28\r\n },\r\n {\r\n \"name\": \"ne_name\",\r\n \"display\": \"NE name\",\r\n \"length\": 28\r\n },\r\n {\r\n \"name\": \"pv_flag\",\r\n \"display\": \"PV flag\",\r\n \"length\": 10\r\n },\r\n {\r\n \"name\": \"ip\",\r\n \"display\": \"IP address\",\r\n \"length\": 32\r\n },\r\n {\r\n \"name\": \"port\",\r\n \"display\": \"Port\",\r\n \"length\": 6\r\n },\r\n {\r\n \"name\": \"status\",\r\n \"display\": \"Status\",\r\n \"length\": 10,\r\n \"alias\": [\r\n \"offline\",\r\n \"online\",\r\n \"standby\",\r\n \"maintain\"\r\n ]\r\n }\r\n ],\r\n \"end\": \"(Number of results = %d)\\n\\n\"\r\n}\r\n'); INSERT INTO `mml_http_map` VALUES (4, 'OMC', 'add', 'neinfo', 'Post', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '', '{\"bodyFmt\":\"PostDB\", \"bodyKey\":\"ne_info\"}', '{\r\n \"retFmt\": \"PostDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"cols\": [\r\n {\r\n \"name\": \"affectedRows\",\r\n \"display\": \"Affected rows\",\r\n \"length\": 11\r\n }\r\n ]\r\n}\r\n'); INSERT INTO `mml_http_map` VALUES (5, 'OMC', 'del', 'neinfo', 'Delete', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '?loc=', '{}', '{\r\n \"retFmt\": \"DeleteDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"cols\": [\r\n {\r\n \"name\": \"affectedRows\",\r\n \"display\": \"Affected rows\",\r\n \"length\": 11\r\n }\r\n ]\r\n}'); INSERT INTO `mml_http_map` VALUES (6, 'OMC', 'mod', 'neinfo', 'Put', '/api/rest/dataManagement/v1/omc_db/ne_info', NULL, NULL, '?loc=', '{\"bodyFmt\":\"PutDB\", \"bodyKey\":\"ne_info\"}', '{\r\n \"retFmt\": \"PutDB\",\r\n \"retMsg\": \"RetCode = %d operation succeeded\\n\\n\",\r\n \"errMsg\": \"ErrorCode = %d operation failed: %s\\n\\n\",\r\n \"cols\": [\r\n {\r\n \"name\": \"affectedRows\",\r\n \"display\": \"Affected rows\",\r\n \"length\": 11\r\n }\r\n ]\r\n}'); From 080290cdaaebf7519f120df60e7040d9c12b10c0 Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Thu, 10 Oct 2024 16:55:20 +0800 Subject: [PATCH 3/5] fix: kpi value change type from int to int64 --- features/pm/performance.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/pm/performance.go b/features/pm/performance.go index 72fea0ea..94aea455 100644 --- a/features/pm/performance.go +++ b/features/pm/performance.go @@ -44,7 +44,7 @@ type KpiReport struct { NeType string `json:"NeType"` KPIs []struct { KPIID string `json:"KPIID"` - Value int `json:"Value"` + Value int64 `json:"Value"` Err string `json:"Err"` } `json:"KPIs"` } `json:"NE"` @@ -62,7 +62,7 @@ type GoldKpi struct { RmUid string `json:"rmUid" xorm:"rm_uid"` NEType string `json:"neType" xorm:"ne_type"` KpiId string `json:"kpiId" xorm:"kpi_id"` - Value int `json:"value"` + Value int64 `json:"value"` Error string `json:"error"` Timestamp string `json:"timestamp"` } From 72ee6a083e64a5f86473a1a8d49f660452a38a17 Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Fri, 11 Oct 2024 10:14:20 +0800 Subject: [PATCH 4/5] add: smsc cdr export to omc --- database/install/sys_dict_data1_i18n_zh.sql | 1 + database/install/sys_dict_data2_i18n_en.sql | 1 + database/install/sys_job.sql | 1 + database/upgrade/upg_sys_dict_data1_i18n_zh.sql | 1 + database/upgrade/upg_sys_dict_data2_i18n_en.sql | 1 + database/upgrade/upg_sys_job.sql | 3 ++- 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/database/install/sys_dict_data1_i18n_zh.sql b/database/install/sys_dict_data1_i18n_zh.sql index 1ae08d32..b69b45d7 100644 --- a/database/install/sys_dict_data1_i18n_zh.sql +++ b/database/install/sys_dict_data1_i18n_zh.sql @@ -689,5 +689,6 @@ INSERT INTO `sys_dict_data` VALUES (2178, 2178, 'dictData.cdr_cause_code.41', ' INSERT INTO `sys_dict_data` VALUES (2179, 2179, 'dictData.cdr_cause_code.42', '拥塞', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2180, 2180, 'dictData.cdr_cause_code.47', '资源不可用', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2181, 2181, 'dictData.cdr_cause_code.50', '请求的设施未订阅', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2182, 2182, 'job.exportSMSCCDR', '定期从短信话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/install/sys_dict_data2_i18n_en.sql b/database/install/sys_dict_data2_i18n_en.sql index f3188862..89b7e292 100644 --- a/database/install/sys_dict_data2_i18n_en.sql +++ b/database/install/sys_dict_data2_i18n_en.sql @@ -689,5 +689,6 @@ INSERT INTO `sys_dict_data` VALUES (4178, 4178, 'dictData.cdr_cause_code.41', 'T INSERT INTO `sys_dict_data` VALUES (4179, 4179, 'dictData.cdr_cause_code.42', 'Congestion', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4180, 4180, 'dictData.cdr_cause_code.47', 'Resources Unavailable Unspec', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4181, 4181, 'dictData.cdr_cause_code.50', 'Requested Facility Not Subscribed', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4182, 4182, 'job.exportSMSCCDR', 'Export regularly from SMSC CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/install/sys_job.sql b/database/install/sys_job.sql index 938765e1..62ac54bf 100644 --- a/database/install/sys_job.sql +++ b/database/install/sys_job.sql @@ -40,5 +40,6 @@ INSERT INTO `sys_job` VALUES (10, 'job.genNeStateAlarm', 'SYSTEM', 'genNeStateAl INSERT INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); INSERT INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); INSERT INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,ne_type,ne_name,rm_uid,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); +INSERT INTO `sys_job` VALUES (14, 'job.exportSMSCCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smsc\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceType\')) as service_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.result\')) as result,DATE_FORMAT(FROM_UNIXTIME(JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.updateTime\'))), \'%Y-%m-%d %H:%i:%s\') as update_time\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smsc_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql index 1a0b62c1..04363e03 100644 --- a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql +++ b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql @@ -696,5 +696,6 @@ REPLACE INTO `sys_dict_data` VALUES (2178, 2178, 'dictData.cdr_cause_code.41', ' REPLACE INTO `sys_dict_data` VALUES (2179, 2179, 'dictData.cdr_cause_code.42', '拥塞', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2180, 2180, 'dictData.cdr_cause_code.47', '资源不可用', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2181, 2181, 'dictData.cdr_cause_code.50', '请求的设施未订阅', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (2182, 2182, 'job.exportSMSCCDR', '定期从短信话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_dict_data2_i18n_en.sql b/database/upgrade/upg_sys_dict_data2_i18n_en.sql index 42437f63..5bb3ed6c 100644 --- a/database/upgrade/upg_sys_dict_data2_i18n_en.sql +++ b/database/upgrade/upg_sys_dict_data2_i18n_en.sql @@ -691,5 +691,6 @@ REPLACE INTO `sys_dict_data` VALUES (4178, 4178, 'dictData.cdr_cause_code.41', ' REPLACE INTO `sys_dict_data` VALUES (4179, 4179, 'dictData.cdr_cause_code.42', 'Congestion', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4180, 4180, 'dictData.cdr_cause_code.47', 'Resources Unavailable Unspec', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4181, 4181, 'dictData.cdr_cause_code.50', 'Requested Facility Not Subscribed', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4182, 4182, 'job.exportSMSCCDR', 'Export regularly from SMSC CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_job.sql b/database/upgrade/upg_sys_job.sql index 00b91c64..df4ca21e 100644 --- a/database/upgrade/upg_sys_job.sql +++ b/database/upgrade/upg_sys_job.sql @@ -43,6 +43,7 @@ REPLACE INTO `sys_job` VALUES (9, 'job.getStateFromNE', 'SYSTEM', 'getStateFromN REPLACE INTO `sys_job` VALUES (10, 'job.genNeStateAlarm', 'SYSTEM', 'genNeStateAlarm', '{\"alarmID\":\"HXEMSSM10000\",\"alarmCode\":10000,\"alarmTitle\":\"The system state is abnormal\",\"neType\":\"OMC\",\"alarmType\":\"EquipmentAlarm\",\"origSeverity\": \"Major\",\"objectName\":\"EMS;SystemManagement;Heartbeat\",\"objectType\":\"SystemState\",\"specificProblem\":\"Alarm cause: the system state of target NE has not been received for {threshold} seconds\", \"specificProblemID\":\"AC10000\",\"threshold\":30}', '0/5 * * * * ?', '3', '0', '0', '0', 'supervisor', 1698478134842, 'admin', 1713781643031, 'job.genNeStateAlarmRemark'); REPLACE INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"sys_log_operate\",\"timeCol\":\"oper_time\",\"timeUnit\":\"milli\",\"columns\":\"oper_id,omc_get_dict_value(title, \\\"i18n_en\\\") as title,business_type,method,request_method,operator_type,oper_name,dept_name,oper_url,oper_ip,oper_location,oper_param,oper_msg,status,oper_time,cost_time\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/operate_log\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724833786290, 'job.exportOperateLog'); REPLACE INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); -REPLACE INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,ne_type,ne_name,rm_uid,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); +REPLACE INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); +REPLACE INTO `sys_job` VALUES (14, 'job.exportSMSCCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smsc\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceType\')) as service_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.result\')) as result,DATE_FORMAT(FROM_UNIXTIME(JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.updateTime\'))), \'%Y-%m-%d %H:%i:%s\') as update_time\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smsc_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); SET FOREIGN_KEY_CHECKS = 1; \ No newline at end of file From 5b29776a97a97e2d85341f1a06f3e3822ad2878f Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Fri, 11 Oct 2024 16:33:35 +0800 Subject: [PATCH 5/5] add: scheduled task to remove exported files --- database/install/sys_dict_data1_i18n_zh.sql | 1 + database/install/sys_dict_data2_i18n_en.sql | 1 + database/install/sys_job.sql | 1 + .../upgrade/upg_sys_dict_data1_i18n_zh.sql | 1 + .../upgrade/upg_sys_dict_data2_i18n_en.sql | 1 + database/upgrade/upg_sys_job.sql | 1 + src/modules/crontask/processor/processor.go | 2 + .../processor/removeFile/removeFile.go | 159 ++++++++++++++++++ 8 files changed, 167 insertions(+) create mode 100644 src/modules/crontask/processor/removeFile/removeFile.go diff --git a/database/install/sys_dict_data1_i18n_zh.sql b/database/install/sys_dict_data1_i18n_zh.sql index b69b45d7..cbb6baf9 100644 --- a/database/install/sys_dict_data1_i18n_zh.sql +++ b/database/install/sys_dict_data1_i18n_zh.sql @@ -690,5 +690,6 @@ INSERT INTO `sys_dict_data` VALUES (2179, 2179, 'dictData.cdr_cause_code.42', ' INSERT INTO `sys_dict_data` VALUES (2180, 2180, 'dictData.cdr_cause_code.47', '资源不可用', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2181, 2181, 'dictData.cdr_cause_code.50', '请求的设施未订阅', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2182, 2182, 'job.exportSMSCCDR', '定期从短信话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2183, 2183, 'job.removeExportedFiles', '定期删除指定目录过期文件', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/install/sys_dict_data2_i18n_en.sql b/database/install/sys_dict_data2_i18n_en.sql index 89b7e292..27eb88a7 100644 --- a/database/install/sys_dict_data2_i18n_en.sql +++ b/database/install/sys_dict_data2_i18n_en.sql @@ -690,5 +690,6 @@ INSERT INTO `sys_dict_data` VALUES (4179, 4179, 'dictData.cdr_cause_code.42', 'C INSERT INTO `sys_dict_data` VALUES (4180, 4180, 'dictData.cdr_cause_code.47', 'Resources Unavailable Unspec', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4181, 4181, 'dictData.cdr_cause_code.50', 'Requested Facility Not Subscribed', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4182, 4182, 'job.exportSMSCCDR', 'Export regularly from SMSC CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4183, 4183, 'job.removeExportedFiles', 'Regularly delete expired files in the specified directory', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/install/sys_job.sql b/database/install/sys_job.sql index 62ac54bf..a41728d4 100644 --- a/database/install/sys_job.sql +++ b/database/install/sys_job.sql @@ -41,5 +41,6 @@ INSERT INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTable INSERT INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); INSERT INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,ne_type,ne_name,rm_uid,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); INSERT INTO `sys_job` VALUES (14, 'job.exportSMSCCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smsc\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceType\')) as service_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.result\')) as result,DATE_FORMAT(FROM_UNIXTIME(JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.updateTime\'))), \'%Y-%m-%d %H:%i:%s\') as update_time\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smsc_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); +INSERT INTO `sys_job` VALUES (15, 'job.removeExportedFiles', 'SYSTEM', 'removeFile', '[{\"filePath\":\"/usr/local/omc/backup/operate_log\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/ims_cdr\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/smf_cdr\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/smsc_cdr\",\"maxDays\":30}]', '0 10 0 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1728634085631, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql index 04363e03..397642b0 100644 --- a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql +++ b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql @@ -697,5 +697,6 @@ REPLACE INTO `sys_dict_data` VALUES (2179, 2179, 'dictData.cdr_cause_code.42', ' REPLACE INTO `sys_dict_data` VALUES (2180, 2180, 'dictData.cdr_cause_code.47', '资源不可用', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2181, 2181, 'dictData.cdr_cause_code.50', '请求的设施未订阅', 'i18n_zh', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2182, 2182, 'job.exportSMSCCDR', '定期从短信话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (2183, 2183, 'job.removeExportedFiles', '定期删除指定目录过期文件', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_dict_data2_i18n_en.sql b/database/upgrade/upg_sys_dict_data2_i18n_en.sql index 5bb3ed6c..5a71db25 100644 --- a/database/upgrade/upg_sys_dict_data2_i18n_en.sql +++ b/database/upgrade/upg_sys_dict_data2_i18n_en.sql @@ -692,5 +692,6 @@ REPLACE INTO `sys_dict_data` VALUES (4179, 4179, 'dictData.cdr_cause_code.42', ' REPLACE INTO `sys_dict_data` VALUES (4180, 4180, 'dictData.cdr_cause_code.47', 'Resources Unavailable Unspec', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4181, 4181, 'dictData.cdr_cause_code.50', 'Requested Facility Not Subscribed', 'i18n_en', '', '', '1', 'supervisor', 1725877564156, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4182, 4182, 'job.exportSMSCCDR', 'Export regularly from SMSC CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4183, 4183, 'job.removeExportedFiles', 'Regularly delete expired files in the specified directory', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); SET FOREIGN_KEY_CHECKS = 1; diff --git a/database/upgrade/upg_sys_job.sql b/database/upgrade/upg_sys_job.sql index df4ca21e..e1293816 100644 --- a/database/upgrade/upg_sys_job.sql +++ b/database/upgrade/upg_sys_job.sql @@ -45,5 +45,6 @@ REPLACE INTO `sys_job` VALUES (11, 'job.exportOperateLog', 'SYSTEM', 'exportTabl REPLACE INTO `sys_job` VALUES (12, 'job.exportIMSCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_ims\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callType\')) as call_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callDuration\')) as call_duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceResult\')) as service_result,DATE_FORMAT(FROM_UNIXTIME(timestamp), \'%Y-%m-%d %H:%i:%s\') AS timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/ims_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1722224659251, ''); REPLACE INTO `sys_job` VALUES (13, 'job.exportSMFCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smf\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) AS record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.chargingID\')) AS charging_id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDType\')) AS subscriber_id_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.subscriberIdentifier.subscriptionIDData\')) AS subscriber_id_data,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.duration\')) AS duration,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.invocationTimestamp\')) as invocationTimestamp,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeUplink\')) AS data_volume_uplink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataVolumeDownlink\')) AS data_volume_downlink,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.listOfMultipleUnitUsage[*].usedUnitContainer[*].dataTotalVolume\')) AS data_total_volume,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.pDUSessionChargingInformation.pDUAddress.pDUIPv4Address\')) AS pdu_ipv4_address,timestamp\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smf_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); REPLACE INTO `sys_job` VALUES (14, 'job.exportSMSCCDR', 'SYSTEM', 'exportTable', '{\"duration\":1,\"tableName\":\"cdr_event_smsc\",\"columns\":\"id,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.recordType\')) as record_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.serviceType\')) as service_type,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.callerParty\')) as caller_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.calledParty\')) as called_party,JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.result\')) as result,DATE_FORMAT(FROM_UNIXTIME(JSON_UNQUOTE(JSON_EXTRACT(cdr_json,\'$.updateTime\'))), \'%Y-%m-%d %H:%i:%s\') as update_time\",\"timeCol\":\"timestamp\",\"timeUnit\":\"second\",\"extras\":\"\",\"filePath\":\"/usr/local/omc/backup/smsc_cdr\"}', '0 0 0/1 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1724309047797, ''); +REPLACE INTO `sys_job` VALUES (15, 'job.removeExportedFiles', 'SYSTEM', 'removeFile', '[{\"filePath\":\"/usr/local/omc/backup/operate_log\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/ims_cdr\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/smf_cdr\",\"maxDays\":30},{\"filePath\":\"/usr/local/omc/backup/smsc_cdr\",\"maxDays\":30}]', '0 10 0 * * ?', '3', '0', '1', '1', 'supervisor', 1698478134842, 'admin', 1728634085631, ''); SET FOREIGN_KEY_CHECKS = 1; \ No newline at end of file diff --git a/src/modules/crontask/processor/processor.go b/src/modules/crontask/processor/processor.go index 09176717..5363e561 100644 --- a/src/modules/crontask/processor/processor.go +++ b/src/modules/crontask/processor/processor.go @@ -10,6 +10,7 @@ import ( "be.ems/src/modules/crontask/processor/getStateFromNE" processorMonitorSysResource "be.ems/src/modules/crontask/processor/monitor_sys_resource" processorNeConfigBackup "be.ems/src/modules/crontask/processor/ne_config_backup" + "be.ems/src/modules/crontask/processor/removeFile" ) // InitCronQueue 初始定时任务队列 @@ -25,4 +26,5 @@ func InitCronQueue() { cron.CreateQueue("getStateFromNE", getStateFromNE.NewProcessor) cron.CreateQueue("genNeStateAlarm", genNeStateAlarm.NewProcessor) cron.CreateQueue("exportTable", exportTable.NewProcessor) + cron.CreateQueue("removeFile", removeFile.NewProcessor) } diff --git a/src/modules/crontask/processor/removeFile/removeFile.go b/src/modules/crontask/processor/removeFile/removeFile.go new file mode 100644 index 00000000..5420b40b --- /dev/null +++ b/src/modules/crontask/processor/removeFile/removeFile.go @@ -0,0 +1,159 @@ +package removeFile + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "time" + + "be.ems/lib/log" + "be.ems/src/framework/cron" +) + +var NewProcessor = &BarProcessor{ + progress: 0, + count: 0, +} + +// bar 队列任务处理 +type BarProcessor struct { + // 任务进度 + progress int + // 执行次数 + count int +} + +type BarParams struct { + FilePath string `json:"filePath"` // file path + MaxDays int `json:"maxDays"` + MaxFiles *int `json:"maxFiles"` // keep max files + MaxSize *int64 `json:"maxSize"` + Extras string `json:"extras"` // extras condition for where +} + +type FileInfo struct { + Path string + Info os.FileInfo +} + +func (s *BarProcessor) Execute(data any) (any, error) { + s.count++ + + options := data.(cron.JobData) + sysJob := options.SysJob + var params []BarParams + + err := json.Unmarshal([]byte(sysJob.TargetParams), ¶ms) + if err != nil { + return nil, err + } + result := []map[string]any{} + for _, param := range params { + res, _ := s.ExecuteOne(param) + result = append(result, res) + } + + // 返回结果，用于记录执行结果 + return map[string]any{ + "result": result, + }, nil +} + +func (s *BarProcessor) ExecuteOne(params BarParams) (map[string]any, error) { + var maxFiles int = 0 + var maxSize int64 = 0 + if params.MaxFiles != nil { + maxFiles = *params.MaxFiles + } + if params.MaxSize != nil { + maxSize = int64(*params.MaxSize * 1024 * 1024) + } + files, err := getFiles(params.FilePath) + if err != nil { + return map[string]any{ + "msg": "failed", + "err": err.Error(), + }, err + } + + // 获取本地时区 + loc, err := time.LoadLocation("Local") + if err != nil { + return map[string]any{ + "msg": "failed", + "err": err.Error(), + }, err + } + cutoff := time.Now().In(loc).AddDate(0, 0, -params.MaxDays) + + var oldFiles []FileInfo + for _, file := range files { + if file.Info.ModTime().Before(cutoff) { + oldFiles = append(oldFiles, file) + } + } + + // 按修改时间排序文件（最旧的在前） + sort.Slice(oldFiles, func(i, j int) bool { + return oldFiles[i].Info.ModTime().Before(oldFiles[j].Info.ModTime()) + }) + + deleted, errorDel := 0, 0 + + // 删除文件，直到满足文件总数不超过maxFiles个且总大小不超过maxSize的条件 + var totalSize int64 + for i, file := range oldFiles { + if (maxFiles > 0 && i >= maxFiles) || (maxSize > 0 && totalSize+file.Info.Size() > maxSize) { + break + } + err := os.Remove(file.Path) + if err != nil { + log.Error("Error deleting file:", file.Path, err) + errorDel++ + continue + } + totalSize += file.Info.Size() + deleted++ + } + + // 如果仍然有超过maxFiles个文件或总大小超过maxSize，继续删除最旧的文件 + remainingFiles := files + sort.Slice(remainingFiles, func(i, j int) bool { + return remainingFiles[i].Info.ModTime().Before(remainingFiles[j].Info.ModTime()) + }) + + for (maxFiles > 0 && len(remainingFiles) > maxFiles) || (maxSize > 0 && totalSize > maxSize) { + file := remainingFiles[0] + err := os.Remove(file.Path) + if err != nil { + log.Error("Error deleting file:", file.Path, err) + remainingFiles = remainingFiles[1:] + continue + } + totalSize -= file.Info.Size() + remainingFiles = remainingFiles[1:] + } + + // 返回结果，用于记录执行结果 + return map[string]any{ + "msg": "successed", + "filePath": params.FilePath, + "deleted": deleted, + "errorDel": errorDel, + }, nil +} + +func getFiles(dir string) ([]FileInfo, error) { + var files []FileInfo + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + files = append(files, FileInfo{Path: path, Info: info}) + } + return nil + }) + return files, err +}