diff --git a/src/modules/trace/service/tcpdump.go b/src/modules/trace/service/tcpdump.go index a0ed1621..2511736d 100644 --- a/src/modules/trace/service/tcpdump.go +++ b/src/modules/trace/service/tcpdump.go @@ -12,6 +12,7 @@ import ( "be.ems/src/framework/logger" "be.ems/src/framework/utils/file" + "be.ems/src/framework/utils/ssh" neService "be.ems/src/modules/network_element/service" ) @@ -32,7 +33,7 @@ var dumpPIDMap sync.Map // DumpStart 触发tcpdump开始抓包 func (s *TCPdump) DumpStart(neType, neId, cmdStr string) (string, error) { // 命令检查 - if strings.Contains(cmdStr, "w") { + if strings.Contains(cmdStr, "-w") { return "", fmt.Errorf("command cannot contain -w") } @@ -53,7 +54,7 @@ func (s *TCPdump) DumpStart(neType, neId, cmdStr string) (string, error) { // bash: tcpdump: command not found msg = strings.TrimSpace(msg) logger.Errorf("DumpStart err: %s => %s", msg, err.Error()) - return "", fmt.Errorf(msg) + return "", fmt.Errorf("%s", msg) } taskCode := time.Now().Format("20060102150405") @@ -81,10 +82,18 @@ func (s *TCPdump) DumpStart(neType, neId, cmdStr string) (string, error) { return "", err } + // 日志文件行号 + PIDMap := s.logFileLastLine(neType, sshClient) + PIDMap["neType"] = neInfo.NeType + PIDMap["neId"] = neInfo.NeId + PIDMap["taskCode"] = taskCode + PIDMap["pid"] = outputPID + PIDMap["cmd"] = sendCmd + // 检查进程 ps aux | grep tcpdump // 强杀 sudo pkill tcpdump pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - dumpPIDMap.Store(pidKey, outputPID) + dumpPIDMap.Store(pidKey, PIDMap) return taskCode, err } @@ -104,12 +113,14 @@ func (s *TCPdump) DumpStop(neType, neId, taskCode string) (string, error) { // 是否存在执行过的进程 pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - pid, ok := dumpPIDMap.Load(pidKey) + PIDMap, ok := dumpPIDMap.Load(pidKey) + if !ok || PIDMap == nil { + return "", fmt.Errorf("tcpdump is not running") + } + pid, ok := PIDMap.(map[string]string)["pid"] if !ok || pid == "" { return "", fmt.Errorf("tcpdump is not running") } - defer dumpPIDMap.Delete(pidKey) - // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) // 命令拼装 @@ -117,12 +128,86 @@ func (s *TCPdump) DumpStop(neType, neId, taskCode string) (string, error) { // pids=$(pgrep -P 1914341) && [ -n "$pids" ] && sudo kill $pids;sudo timeout 2s cat tcpdump.log output, err := sshClient.RunCMD(sendCmd) if err != nil || strings.HasPrefix(output, "stderr:") { - logger.Warnf("DumpStop err: %s => %s", strings.TrimSpace(output), err.Error()) + logger.Errorf("DumpStop err: %s => %s", strings.TrimSpace(output), err.Error()) return "", err } + + s.logFileLastLineToFile(PIDMap.(map[string]string), sshClient) + dumpPIDMap.Delete(pidKey) return output, nil } +// logFileLastLine 日志文件最后行号 +func (s *TCPdump) logFileLastLine(neType string, sshClient *ssh.ConnSSH) map[string]string { + logFileArr := make([]string, 0) + mapFile := make(map[string]string, 0) + + // 存放文件目录 /var/log/xxx.log + if neType == "IMS" { + logFileArr = append(logFileArr, + "/var/log/ims/pcscf/pcscf.log", + "/var/log/ims/bgcf/bgcf.log", + "/var/log/ims/bsf/bsf.log", + "/var/log/ims/icscf/icscf.log", + "/var/log/ims/ismc/ismc.log", + "/var/log/ims/mmtel/mmtel.log", + "/var/log/ims/scscf/scscf.log", + "/var/log/ims/iwf/iwf.log", + ) + } else { + neLogFile := fmt.Sprintf("/var/log/%s.log", strings.ToLower(neType)) + logFileArr = append(logFileArr, neLogFile) + } + + for _, v := range logFileArr { + lastLine, err := sshClient.RunCMD(fmt.Sprintf("sed -n '$=' %s", v)) + lastLine = strings.TrimSpace(lastLine) + if err != nil || strings.HasPrefix(lastLine, "sed: can't") { + logger.Errorf("logFileLastLine err: %s => %s", lastLine, err.Error()) + continue + } + mapFile[v] = lastLine + } + return mapFile +} + +// logFileLastLine 日志文件最后行号 +func (s *TCPdump) logFileLastLineToFile(PIDMap map[string]string, sshClient *ssh.ConnSSH) error { + // 网元主机的SSH客户端进行文件传输 + sftpClient, err := sshClient.NewClientSFTP() + if err != nil { + return fmt.Errorf("ne info sftp client err") + } + defer sftpClient.Close() + + neType := PIDMap["neType"] + neId := PIDMap["neId"] + taskCode := PIDMap["taskCode"] + // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 + sshClient.RunCMD("mkdir -p /tmp/omc && sudo chmod 777 -R /tmp/omc") + neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neType), neId, taskCode) + + lastLineMap := s.logFileLastLine(neType, sshClient) + for lastLogFile, lastFileLine := range lastLineMap { + for startLogFile, startFileLine := range PIDMap { + if lastLogFile == startLogFile && lastFileLine != "" { + if startFileLine == "" { + startFileLine = "1" // 起始行号从第一行开始 + } + outputFile := fmt.Sprintf("%s/%s", neDirTemp, filepath.Base(lastLogFile)) + sendCmd := fmt.Sprintf("sed -n \"%s,%sp\" \"%s\" > \"%s\"", startFileLine, lastFileLine, lastLogFile, outputFile) + // sed -n "1,5p" "/var/log/amf.log" > "/tmp/omc/tcpdump/amf/001/20241008141336/amf.log" + output, err := sshClient.RunCMD(sendCmd) + if err != nil || strings.HasPrefix(output, "stderr:") { + logger.Errorf("logFileLastLineToFile err: %s => %s", strings.TrimSpace(output), err.Error()) + continue + } + } + } + } + return nil +} + // DumpDownload 抓包文件网元端复制到本地输出zip文件 func (s *TCPdump) DumpDownload(neType, neId, taskCode string) (string, error) { // 查询网元获取IP