From eec27d039a0e9dcfa136fd05a98e27488ca0c2bb Mon Sep 17 00:00:00 2001 From: simonzhangsz Date: Tue, 27 Aug 2024 20:11:47 +0800 Subject: [PATCH] add: list and download log&cdr export file --- database/install/sys_dict_data1_i18n_zh.sql | 4 + database/install/sys_dict_data2_i18n_en.sql | 4 + .../upgrade/upg_sys_dict_data1_i18n_zh.sql | 5 +- .../upgrade/upg_sys_dict_data2_i18n_en.sql | 4 + features/features.go | 4 +- features/lm/file_export/controller.go | 124 ++++++++++++++++++ features/lm/file_export/model.go | 34 +++++ features/lm/file_export/route.go | 40 ++++++ features/lm/service.go | 17 +++ lib/file/file_linux.go | 84 ++++++++++++ lib/file/file_windows.go | 82 ++++++++++++ 11 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 features/lm/file_export/controller.go create mode 100644 features/lm/file_export/model.go create mode 100644 features/lm/file_export/route.go create mode 100644 features/lm/service.go create mode 100644 lib/file/file_linux.go create mode 100644 lib/file/file_windows.go diff --git a/database/install/sys_dict_data1_i18n_zh.sql b/database/install/sys_dict_data1_i18n_zh.sql index 3faf433b..8c9ea585 100644 --- a/database/install/sys_dict_data1_i18n_zh.sql +++ b/database/install/sys_dict_data1_i18n_zh.sql @@ -667,6 +667,10 @@ INSERT INTO `sys_dict_data` VALUES (2156, 2156, 'job.ne_config_backup_remark', ' INSERT INTO `sys_dict_data` VALUES (2157, 2157, 'job.exportOperateLog', '定期从操作日志表导出文件到指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2158, 2158, 'job.exportIMSCDR', '定期从语音话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (2159, 2159, 'job.exportSMFCDR', '定期从数据话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2160, 2160, 'table.sys_log_operate', '操作日志', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2161, 2161, 'table.cdr_event_ims', '语音话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2162, 2162, 'table.cdr_event_smf', '数据话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (2163, 2163, 'table.cdr_event_smsc', '短信话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); -- 多租户 INSERT INTO `sys_dict_data` VALUES (11000, 11000, 'menu.security.tenant', '租户管理', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL); diff --git a/database/install/sys_dict_data2_i18n_en.sql b/database/install/sys_dict_data2_i18n_en.sql index e09502b8..04e83267 100644 --- a/database/install/sys_dict_data2_i18n_en.sql +++ b/database/install/sys_dict_data2_i18n_en.sql @@ -667,6 +667,10 @@ INSERT INTO `sys_dict_data` VALUES (4156, 4156, 'job.ne_config_backup_remark', ' INSERT INTO `sys_dict_data` VALUES (4157, 4157, 'job.exportOperateLog', 'Export regularly from operation log table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4158, 4158, 'job.exportIMSCDR', 'Export regularly from IMS CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); INSERT INTO `sys_dict_data` VALUES (4159, 4159, 'job.exportSMFCDR', 'Export regularly from SMF CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4160, 4160, 'table.sys_log_operate', 'Operation Log', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4161, 4161, 'table.cdr_event_ims', 'Voice CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4162, 4162, 'table.cdr_event_smf', 'Data CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +INSERT INTO `sys_dict_data` VALUES (4163, 4163, 'table.cdr_event_smsc', 'SMS CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); -- 多租户 INSERT INTO `sys_dict_data` VALUES (14000, 14000, 'menu.security.tenant', 'Tenant Management', 'i18n_en', '', '', '1', 'supervisor', 1705550000000, '', 0, ''); diff --git a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql index 74efc7fd..b80beed3 100644 --- a/database/upgrade/upg_sys_dict_data1_i18n_zh.sql +++ b/database/upgrade/upg_sys_dict_data1_i18n_zh.sql @@ -674,7 +674,10 @@ REPLACE INTO `sys_dict_data` VALUES (2156, 2156, 'job.ne_config_backup_remark', REPLACE INTO `sys_dict_data` VALUES (2157, 2157, 'job.exportOperateLog', '定期从操作日志表导出文件到指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2158, 2158, 'job.exportIMSCDR', '定期从语音话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (2159, 2159, 'job.exportSMFCDR', '定期从数据话单表导出文件至指定目录', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); - +REPLACE INTO `sys_dict_data` VALUES (2160, 2160, 'table.sys_log_operate', '操作日志', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (2161, 2161, 'table.cdr_event_ims', '语音话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (2162, 2162, 'table.cdr_event_smf', '数据话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (2163, 2163, 'table.cdr_event_smsc', '短信话单', 'i18n_zh', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); -- multi-tenancy REPLACE INTO `sys_dict_data` VALUES (11000, 11000, 'menu.security.tenant', '租户管理', 'i18n_zh', NULL, NULL, '1', 'supervisor', 1700000000000, NULL, 0, NULL); diff --git a/database/upgrade/upg_sys_dict_data2_i18n_en.sql b/database/upgrade/upg_sys_dict_data2_i18n_en.sql index 442ed4d0..02db1429 100644 --- a/database/upgrade/upg_sys_dict_data2_i18n_en.sql +++ b/database/upgrade/upg_sys_dict_data2_i18n_en.sql @@ -669,6 +669,10 @@ REPLACE INTO `sys_dict_data` VALUES (4156, 4156, 'job.ne_config_backup_remark', REPLACE INTO `sys_dict_data` VALUES (4157, 4157, 'job.exportOperateLog', 'Export regularly from operation log table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4158, 4158, 'job.exportIMSCDR', 'Export regularly from IMS CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); REPLACE INTO `sys_dict_data` VALUES (4159, 4159, 'job.exportSMFCDR', 'Export regularly from SMF CDR table', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4160, 4160, 'table.sys_log_operate', 'Operation Log', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4161, 4161, 'table.cdr_event_ims', 'Voice CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4162, 4162, 'table.cdr_event_smf', 'Data CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); +REPLACE INTO `sys_dict_data` VALUES (4163, 4163, 'table.cdr_event_smsc', 'SMS CDR', 'i18n_en', '', '', '1', 'supervisor', 1721902269805, '', 0, ''); -- 多租户 diff --git a/features/features.go b/features/features.go index 068beb6a..70510558 100644 --- a/features/features.go +++ b/features/features.go @@ -1,6 +1,7 @@ package features import ( + "be.ems/features/lm" "be.ems/features/pm" "be.ems/lib/log" "github.com/gin-gonic/gin" @@ -10,8 +11,9 @@ func InitServiceEngine(r *gin.Engine) { log.Info("======init feature group gin.Engine") // featuresGroup := r.Group("/") - // 注册 PM 模块的路由 + // 注册 各个features 模块的路由 pm.InitSubServiceRoute(r) + lm.InitSubServiceRoute(r) // return featuresGroup } diff --git a/features/lm/file_export/controller.go b/features/lm/file_export/controller.go new file mode 100644 index 00000000..18648420 --- /dev/null +++ b/features/lm/file_export/controller.go @@ -0,0 +1,124 @@ +package file_export + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + + "be.ems/lib/file" + "be.ems/lib/log" + "be.ems/lib/services" + "be.ems/src/framework/datasource" + "be.ems/src/framework/i18n" + "be.ems/src/framework/utils/ctx" + "github.com/gin-gonic/gin" +) + +type SysJobResponse struct { + SysJob + TableName string `json:"tableName"` + TableDisplay string `json:"tableDisplay"` + FilePath string `json:"filePath"` +} + +type TargetParams struct { + Duration int `json:"duration"` + TableName string `json:"tableName"` + Columns string `json:"columns"` // exported column name of time string + TimeCol string `json:"timeCol"` // time stamp of column name + TimeUnit string `json:"timeUnit"` // timestamp unit: second/micro/milli + Extras string `json:"extras"` // extras condition for where + FilePath string `json:"filePath"` // file path +} + +func (m *SysJob) GetFileExportTable(c *gin.Context) { + var results []SysJob + + err := datasource.DefaultDB().Table(m.TableName()).Where("invoke_target=? and status=1", INVOKE_FILE_EXPORT). + Find(&results).Error + if err != nil { + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + return + } + language := ctx.AcceptLanguage(c) + var response []SysJobResponse + for _, job := range results { + var params TargetParams + if err := json.Unmarshal(job.TargetParams, ¶ms); err != nil { + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + return + } + TableDisplay := i18n.TKey(language, "table."+params.TableName) + if TableDisplay == "" { + TableDisplay = params.TableName + } + response = append(response, SysJobResponse{ + SysJob: job, + TableName: params.TableName, + TableDisplay: TableDisplay, + FilePath: params.FilePath, + }) + } + c.JSON(http.StatusOK, services.DataResp(response)) +} + +func (m *FileExport) GetFileList(c *gin.Context) { + dir := c.Query("path") + suffix := c.Query("suffix") + + files, err := file.GetFileInfo(dir, suffix) + if err != nil { + log.Error(err) + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + } + total := int64(len(files)) + c.JSON(http.StatusOK, services.TotalDataResp(files, total)) +} + +func (m *FileExport) Total(c *gin.Context) { + dir := c.Query("path") + + fileCount, dirCount, err := file.GetFileAndDirCount(dir) + if err != nil { + log.Error(err) + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + return + } + total := fileCount + dirCount + c.JSON(http.StatusOK, services.TotalResp(int64(total))) +} + +func (m *FileExport) DownloadHandler(c *gin.Context) { + dir := c.Query("path") + fileName := c.Param("fileName") + filePath := filepath.Join(dir, fileName) + + file, err := os.Open(filePath) + if err != nil { + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + return + } + defer file.Close() + + if _, err := os.Stat(filePath); os.IsNotExist(err) { + c.JSON(http.StatusNotFound, services.ErrResp(err.Error())) + return + } + + c.Header("Content-Disposition", "attachment; filename="+fileName) + c.Header("Content-Type", "application/octet-stream") + c.File(filePath) +} + +func (m *FileExport) Delete(c *gin.Context) { + fileName := c.Param("fileName") + dir := c.Query("path") + filePath := filepath.Join(dir, fileName) + + if err := os.Remove(filePath); err != nil { + c.JSON(http.StatusInternalServerError, services.ErrResp(err.Error())) + return + } + c.JSON(http.StatusNoContent, nil) // 204 No Content +} diff --git a/features/lm/file_export/model.go b/features/lm/file_export/model.go new file mode 100644 index 00000000..83613c29 --- /dev/null +++ b/features/lm/file_export/model.go @@ -0,0 +1,34 @@ +package file_export + +import ( + "gorm.io/datatypes" +) + +const ( + INVOKE_FILE_EXPORT = "exportTable" +) + +type SysJob struct { + JobID int64 `gorm:"column:job_id;primary_key;auto_increment" json:"job_id"` //任务ID + InvokeTarget string `gorm:"column:invoke_target" json:"invoke_target"` //调用目标字符串 + TargetParams datatypes.JSON `gorm:"column:target_params;type:json" json:"target_params,omitempty"` //调用目标传入参数 +} + +func (m *SysJob) TableName() string { + return "sys_job" +} + +type FileExport struct { + FileType string `json:"fileType"` // 文件类型 + FileMode string `json:"fileMode"` // 文件的权限 + LinkCount int64 `json:"linkCount"` // 硬链接数目 + Owner string `json:"owner"` // 所属用户 + Group string `json:"group"` // 所属组 + Size int64 `json:"size"` // 文件的大小 + ModifiedTime int64 `json:"modifiedTime"` // 最后修改时间，单位为秒 + FileName string `json:"fileName"` // 文件的名称 +} + +func (m *FileExport) TableName() string { + return "file_export" +} diff --git a/features/lm/file_export/route.go b/features/lm/file_export/route.go new file mode 100644 index 00000000..d6caba91 --- /dev/null +++ b/features/lm/file_export/route.go @@ -0,0 +1,40 @@ +package file_export + +import ( + "be.ems/src/framework/middleware" + "github.com/gin-gonic/gin" +) + +// Register Routes for file_export +func Register(r *gin.RouterGroup) { + + lmTable := r.Group("/table") + { + var m *SysJob + lmTable.GET("/list", + middleware.PreAuthorize(nil), + m.GetFileExportTable, + ) + + } + lmFile := r.Group("/file") + { + var f *FileExport + lmFile.GET("/list", + middleware.PreAuthorize(nil), + f.GetFileList, + ) + lmFile.GET("/total", + middleware.PreAuthorize(nil), + f.Total, + ) + lmFile.GET("/:fileName", + middleware.PreAuthorize(nil), + f.DownloadHandler, + ) + lmFile.DELETE("/:fileName", + middleware.PreAuthorize(nil), + f.Delete, + ) + } +} diff --git a/features/lm/service.go b/features/lm/service.go new file mode 100644 index 00000000..5cdaa943 --- /dev/null +++ b/features/lm/service.go @@ -0,0 +1,17 @@ +// log management package + +package lm + +import ( + "be.ems/features/lm/file_export" + "be.ems/lib/log" + "github.com/gin-gonic/gin" +) + +func InitSubServiceRoute(r *gin.Engine) { + log.Info("======init Log management group gin.Engine") + + lmGroup := r.Group("/lm") + // register sub modules routes + file_export.Register(lmGroup) +} diff --git a/lib/file/file_linux.go b/lib/file/file_linux.go new file mode 100644 index 00000000..6e5f4640 --- /dev/null +++ b/lib/file/file_linux.go @@ -0,0 +1,84 @@ +//go:build linux +// +build linux + +package file + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +type FileInfo struct { + FileType string `json:"fileType"` // 文件类型 + FileMode string `json:"fileMode"` // 文件的权限 + LinkCount int64 `json:"linkCount"` // 硬链接数目 + Owner string `json:"owner"` // 所属用户 + Group string `json:"group"` // 所属组 + Size int64 `json:"size"` // 文件的大小 + ModifiedTime int64 `json:"modifiedTime"` // 最后修改时间，单位为秒 + FileName string `json:"fileName"` // 文件的名称 +} + +func GetFileInfo(dir, suffix string) ([]FileInfo, error) { + var files []FileInfo + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == dir { + return nil // 跳过当前目录 + } + + fileType := "file" + if info.IsDir() { + fileType = "directory" + } else if info.Mode()&os.ModeSymlink != 0 { + fileType = "symlink" + } + + // check if match suffix + if (suffix != "" && filepath.Ext(path) == suffix) || suffix == "" { + fileInfo := FileInfo{ + FileType: fileType, + FileMode: info.Mode().String(), + LinkCount: int64(info.Sys().(*syscall.Stat_t).Nlink), + Owner: fmt.Sprintf("%d", info.Sys().(*syscall.Stat_t).Uid), + Group: fmt.Sprintf("%d", info.Sys().(*syscall.Stat_t).Gid), + Size: info.Size(), + ModifiedTime: info.ModTime().Unix(), + FileName: info.Name(), + } + files = append(files, fileInfo) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +func GetFileAndDirCount(dir string) (int, int, error) { + var fileCount, dirCount int + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == dir { + return nil // 跳过当前目录 + } + if info.IsDir() { + dirCount++ + } else { + fileCount++ + } + return nil + }) + + return fileCount, dirCount, err +} diff --git a/lib/file/file_windows.go b/lib/file/file_windows.go new file mode 100644 index 00000000..b19e37a3 --- /dev/null +++ b/lib/file/file_windows.go @@ -0,0 +1,82 @@ +//go:build windows +// +build windows + +package file + +import ( + "os" + "path/filepath" +) + +type FileInfo struct { + FileType string `json:"fileType"` // 文件类型 + FileMode string `json:"fileMode"` // 文件的权限 + LinkCount int64 `json:"linkCount"` // 硬链接数目 + Owner string `json:"owner"` // 所属用户 + Group string `json:"group"` // 所属组 + Size int64 `json:"size"` // 文件的大小 + ModifiedTime int64 `json:"modifiedTime"` // 最后修改时间，单位为秒 + FileName string `json:"fileName"` // 文件的名称 +} + +func GetFileInfo(dir, suffix string) ([]FileInfo, error) { + var files []FileInfo + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == dir { + return nil // 跳过当前目录 + } + + fileType := "file" + if info.IsDir() { + fileType = "directory" + } else if info.Mode()&os.ModeSymlink != 0 { + fileType = "symlink" + } + + // check if match suffix + if (suffix != "" && filepath.Ext(path) == suffix) || suffix == "" { + fileInfo := FileInfo{ + FileType: fileType, + FileMode: info.Mode().String(), + LinkCount: 0, + Owner: "N/A", + Group: "N/A", + Size: info.Size(), + ModifiedTime: info.ModTime().Unix(), + FileName: info.Name(), + } + files = append(files, fileInfo) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +func GetFileAndDirCount(dir string) (int, int, error) { + var fileCount, dirCount int + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == dir { + return nil // 跳过当前目录 + } + if info.IsDir() { + dirCount++ + } else { + fileCount++ + } + return nil + }) + + return fileCount, dirCount, err +}