diff --git a/src/modules/trace/controller/tcpdump.go b/src/modules/trace/controller/tcpdump.go index 94012723..0f0d7bd0 100644 --- a/src/modules/trace/controller/tcpdump.go +++ b/src/modules/trace/controller/tcpdump.go @@ -15,7 +15,7 @@ import ( // 实例化控制层 TcpdumpController 结构体 var NewTcpdump = &TcpdumpController{ - TcpdumpService: traceService.NewTcpdumpImpl, + TcpdumpService: traceService.NewTCPdump, neInfoService: neService.NewNeInfoImpl, } @@ -24,7 +24,7 @@ var NewTcpdump = &TcpdumpController{ // PATH /tcpdump type TcpdumpController struct { // 信令抓包服务 - TcpdumpService traceService.ITcpdump + TcpdumpService *traceService.TCPdump // 网元信息服务 neInfoService neService.INeInfo } diff --git a/src/modules/trace/service/tcpdump.go b/src/modules/trace/service/tcpdump.go index 9f77fb06..494726b2 100644 --- a/src/modules/trace/service/tcpdump.go +++ b/src/modules/trace/service/tcpdump.go @@ -1,16 +1,248 @@ package service -// 信令抓包 服务层接口 -type ITcpdump interface { - // DumpStart 触发tcpdump开始抓包 - DumpStart(neType, neId, cmdStr string) (string, error) +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "time" - // DumpStop 停止已存在抓包句柄 - DumpStop(neType, neId, taskCode string) (string, error) + "be.ems/src/framework/logger" + "be.ems/src/framework/utils/file" + neService "be.ems/src/modules/network_element/service" +) - // DumpDownload 抓包文件网元端复制到本地输出zip文件 - DumpDownload(neType, neId, taskCode string) (string, error) - - // UPFTrace UPF标准版内部抓包 - UPFTrace(neType, neId, cmdStr string) (string, error) +// 实例化服务层 TCPdump 结构体 +var NewTCPdump = &TCPdump{ + neInfoService: neService.NewNeInfoImpl, +} + +// 信令抓包 服务层处理 +type TCPdump struct { + // 网元信息服务 + neInfoService neService.INeInfo +} + +// 抓包进程PID +var dumpPIDMap sync.Map + +// DumpStart 触发tcpdump开始抓包 +func (s *TCPdump) DumpStart(neType, neId, cmdStr string) (string, error) { + // 命令检查 + if strings.Contains(cmdStr, "w") { + return "", fmt.Errorf("command cannot contain -w") + } + + // 查询网元获取IP + neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) + if neInfo.NeId != neId || neInfo.IP == "" { + return "", fmt.Errorf("app.common.noNEInfo") + } + // 网元主机的SSH客户端 + sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) + if err != nil { + return "", err + } + defer sshClient.Close() + + // 检查是否安装tcpdump + if msg, err := sshClient.RunCMD("sudo tcpdump --version"); err != nil { + // bash: tcpdump: command not found + msg = strings.TrimSpace(msg) + logger.Errorf("DumpStart err: %s => %s", msg, err.Error()) + return "", fmt.Errorf(msg) + } + + taskCode := time.Now().Format("20060102150405") + // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 + neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) + sshClient.RunCMD(fmt.Sprintf("mkdir -p %s && sudo chmod 777 -R /tmp/omc", neDirTemp)) + + // 命令拼装 + logPath := fmt.Sprintf("%s/tcpdump.log", neDirTemp) + filePath := fmt.Sprintf("%s/part_%s.pcap ", neDirTemp, taskCode) + if strings.Contains(cmdStr, "-G") { + filePath = fmt.Sprintf("%s/part_%%Y%%m%%d%%H%%M%%S.pcap ", neDirTemp) + } + sendCmd := fmt.Sprintf("sudo timeout 60m sudo tcpdump -i any %s -w %s > %s 2>&1 & echo $!", cmdStr, filePath, logPath) + // sudo timeout 60m sudo tcpdump -i any -n -s 0 -v -G 60 -W 6 -w /tmp/omc/tcpdump/udm/001/20240817104241/part_%Y-%m-%d_%H:%M:%S.pcap > /tmp/omc/tcpdump/udm/001/20240817104241/tcpdump.log 2>&1 & echo $! + // sudo timeout 60m sudo tcpdump -i any -n -s 0 -v -w /tmp/omc/tcpdump/udm/001/20240817105440/part_2024-08-17_10:54:40.pcap > /tmp/omc/tcpdump/udm/001/20240817105440/tcpdump.log 2>&1 & echo $! + // + // timeout 超时60分钟后发送kill命令，1分钟后强制终止命令。tcpdump -G 文件轮转间隔时间（秒） -W 文件轮转保留最近数量 + // sudo timeout --kill-after=1m 60m sudo tcpdump -i any -n -s 0 -v -G 10 -W 7 -w /tmp/part_%Y%m%d%H%M%S.pcap > /tmp/part.log 2>&1 & echo $! + // sudo kill $(pgrep -P 722729) + outputPID, err := sshClient.RunCMD(sendCmd) + outputPID = strings.TrimSpace(outputPID) + if err != nil || strings.HasPrefix(outputPID, "stderr:") { + logger.Errorf("DumpStart err: %s => %s", outputPID, err.Error()) + return "", err + } + + // 检查进程 ps aux | grep tcpdump + // 强杀 sudo pkill tcpdump + pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) + dumpPIDMap.Store(pidKey, outputPID) + return taskCode, err +} + +// DumpStop 停止已存在抓包句柄 +func (s *TCPdump) DumpStop(neType, neId, taskCode string) (string, error) { + // 查询网元获取IP + neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) + if neInfo.NeId != neId || neInfo.IP == "" { + return "", fmt.Errorf("app.common.noNEInfo") + } + // 网元主机的SSH客户端 + sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) + if err != nil { + return "", err + } + defer sshClient.Close() + + // 是否存在执行过的进程 + pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) + pid, ok := dumpPIDMap.Load(pidKey) + if !ok || pid == "" { + return "", fmt.Errorf("tcpdump is not running") + } + defer dumpPIDMap.Delete(pidKey) + + // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 + neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) + // 命令拼装 + sendCmd := fmt.Sprintf("pids=$(pgrep -P %s) && [ -n \"$pids\" ] && sudo kill $pids;sudo timeout 2s cat %s/tcpdump.log", pid, neDirTemp) + // pids=$(pgrep -P 1914341) && [ -n "$pids" ] && sudo kill $pids;sudo timeout 2s cat tcpdump.log + output, err := sshClient.RunCMD(sendCmd) + if err != nil || strings.HasPrefix(output, "stderr:") { + logger.Warnf("DumpStop err: %s => %s", strings.TrimSpace(output), err.Error()) + return "", err + } + return output, nil +} + +// DumpDownload 抓包文件网元端复制到本地输出zip文件 +func (s *TCPdump) DumpDownload(neType, neId, taskCode string) (string, error) { + // 查询网元获取IP + neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) + if neInfo.NeId != neId || neInfo.IP == "" { + return "", fmt.Errorf("app.common.noNEInfo") + } + // 网元主机的SSH客户端 + sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) + if err != nil { + return "", err + } + defer sshClient.Close() + // 网元主机的SSH客户端进行文件传输 + sftpClient, err := sshClient.NewClientSFTP() + if err != nil { + return "", fmt.Errorf("ne info sftp client err") + } + defer sftpClient.Close() + + neTypeLower := strings.ToLower(neInfo.NeType) + // 网管本地路径 + localDirPath := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s", neTypeLower, neInfo.NeId) + if runtime.GOOS == "windows" { + localDirPath = fmt.Sprintf("C:%s", localDirPath) + } + + // 网元pcap目录 /tmp/omc/tcpdump/udm/001/20240817104241 + sshClient.RunCMD("mkdir -p /tmp/omc && sudo chmod 777 -R /tmp/omc") + neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", neTypeLower, neInfo.NeId, taskCode) + // 网元端复制到本地 + localDirFilePath := filepath.Join(localDirPath, taskCode) + if err = sftpClient.CopyDirRemoteToLocal(neDirTemp, localDirFilePath); err != nil { + return "", fmt.Errorf("copy tcpdump file err") + } + + // 压缩zip文件名 + zipFileName := fmt.Sprintf("%s-%s-pcap-%s.zip", neTypeLower, neInfo.NeId, taskCode) + zipFilePath := filepath.Join(localDirPath, zipFileName) + if err := file.CompressZipByDir(zipFilePath, localDirFilePath); err != nil { + return "", fmt.Errorf("compress zip err") + } + + _ = os.RemoveAll(localDirFilePath) // 删除本地临时目录 + return zipFilePath, nil +} + +// UPFTrace UPF标准版内部抓包 +func (s *TCPdump) UPFTrace(neType, neId, cmdStr string) (string, error) { + // 命令检查 + if strings.Contains(cmdStr, "file") { + return "", fmt.Errorf("command cannot contain file") + } + + // 查询网元获取IP + neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) + if neInfo.NeId != neId || neInfo.IP == "" { + return "", fmt.Errorf("app.common.noNEInfo") + } + // 网元主机的SSH客户端 + sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) + if err != nil { + return "", err + } + defer sshClient.Close() + // 网元主机的Telnet客户端 + telnetClient, err := s.neInfoService.NeRunTelnetClient("UPF", neInfo.NeId, 2) + if err != nil { + return "", err + } + defer telnetClient.Close() + + // 命令拼装 + fileName := fmt.Sprintf("%s_%s_part_%s.pcap ", neInfo.NeType, neInfo.NeId, time.Now().Format("20060102150405")) + pcapCmd := fmt.Sprintf("%s\r\n", cmdStr) + // 以off结尾是停止抓包，不需要写文件 + if !strings.Contains(cmdStr, "off") { + // pcap trace rx tx max 100000 intfc any file UPF_001_part_20240817164516.pcap + pcapCmd = fmt.Sprintf("%s file %s\r\n", cmdStr, fileName) + } + // 发送命令 UPF内部默认输出路径/tmp只能写文件名 + // pcap trace rx tx max 100000 intfc any file upf_test.pcap + // pcap trace rx tx off + output, err := telnetClient.RunCMD(pcapCmd) + if err != nil { + logger.Warnf("DumpUPF err: %s => %s", output, err.Error()) + return "", err + } + + // 结果截取 + arr := strings.Split(output, "\r\n") + if len(arr) == 2 { + return "", fmt.Errorf("trace pacp run failed") + } + if len(arr) > 3 { + resMsg := arr[2] + // pcap trace: unknown input `f file UPF_001_part_2024-08-19...' + // pcap trace: dispatch trace already enabled... + // pcap trace: dispatch trace already disabled... + // pcap trace: No packets captured... + // Write 100000 packets to /tmp/UPF_001_part_20240817164516.pcap, and stop capture... + if strings.Contains(resMsg, "unknown input") { + return "", fmt.Errorf("trace pacp unknown input") + } + if strings.Contains(resMsg, "already enabled") { + return "", fmt.Errorf("trace pacp already running") + } + if strings.Contains(resMsg, "already disabled") { + return "", fmt.Errorf("trace pacp not running") + } + if strings.Contains(resMsg, "No packets") { + return "", fmt.Errorf("trace pacp not packets") + } + if strings.Contains(resMsg, "packets to") { + matches := regexp.MustCompile(`(/tmp/[^,\s]+)`).FindStringSubmatch(resMsg) + if len(matches) == 0 { + return "", fmt.Errorf("file path not found") + } + return matches[0], nil + } + } + return "trace pacp running", nil } diff --git a/src/modules/trace/service/tcpdump.impl.go b/src/modules/trace/service/tcpdump.impl.go deleted file mode 100644 index 72194832..00000000 --- a/src/modules/trace/service/tcpdump.impl.go +++ /dev/null @@ -1,248 +0,0 @@ -package service - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "runtime" - "strings" - "sync" - "time" - - "be.ems/src/framework/logger" - "be.ems/src/framework/utils/file" - neService "be.ems/src/modules/network_element/service" -) - -// 实例化服务层 TcpdumpImpl 结构体 -var NewTcpdumpImpl = &TcpdumpImpl{ - neInfoService: neService.NewNeInfoImpl, -} - -// 信令抓包 服务层处理 -type TcpdumpImpl struct { - // 网元信息服务 - neInfoService neService.INeInfo -} - -// 抓包进程PID -var dumpPIDMap sync.Map - -// DumpStart 触发tcpdump开始抓包 -func (s *TcpdumpImpl) DumpStart(neType, neId, cmdStr string) (string, error) { - // 命令检查 - if strings.Contains(cmdStr, "w") { - return "", fmt.Errorf("command cannot contain -w") - } - - // 查询网元获取IP - neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) - if neInfo.NeId != neId || neInfo.IP == "" { - return "", fmt.Errorf("app.common.noNEInfo") - } - // 网元主机的SSH客户端 - sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) - if err != nil { - return "", err - } - defer sshClient.Close() - - // 检查是否安装tcpdump - if msg, err := sshClient.RunCMD("sudo tcpdump --version"); err != nil { - // bash: tcpdump: command not found - msg = strings.TrimSpace(msg) - logger.Errorf("DumpStart err: %s => %s", msg, err.Error()) - return "", fmt.Errorf(msg) - } - - taskCode := time.Now().Format("20060102150405") - // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 - neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - sshClient.RunCMD(fmt.Sprintf("mkdir -p %s && sudo chmod 777 -R /tmp/omc", neDirTemp)) - - // 命令拼装 - logPath := fmt.Sprintf("%s/tcpdump.log", neDirTemp) - filePath := fmt.Sprintf("%s/part_%s.pcap ", neDirTemp, taskCode) - if strings.Contains(cmdStr, "-G") { - filePath = fmt.Sprintf("%s/part_%%Y%%m%%d%%H%%M%%S.pcap ", neDirTemp) - } - sendCmd := fmt.Sprintf("sudo timeout 60m sudo tcpdump -i any %s -w %s > %s 2>&1 & echo $!", cmdStr, filePath, logPath) - // sudo timeout 60m sudo tcpdump -i any -n -s 0 -v -G 60 -W 6 -w /tmp/omc/tcpdump/udm/001/20240817104241/part_%Y-%m-%d_%H:%M:%S.pcap > /tmp/omc/tcpdump/udm/001/20240817104241/tcpdump.log 2>&1 & echo $! - // sudo timeout 60m sudo tcpdump -i any -n -s 0 -v -w /tmp/omc/tcpdump/udm/001/20240817105440/part_2024-08-17_10:54:40.pcap > /tmp/omc/tcpdump/udm/001/20240817105440/tcpdump.log 2>&1 & echo $! - // - // timeout 超时60分钟后发送kill命令，1分钟后强制终止命令。tcpdump -G 文件轮转间隔时间（秒） -W 文件轮转保留最近数量 - // sudo timeout --kill-after=1m 60m sudo tcpdump -i any -n -s 0 -v -G 10 -W 7 -w /tmp/part_%Y%m%d%H%M%S.pcap > /tmp/part.log 2>&1 & echo $! - // sudo kill $(pgrep -P 722729) - outputPID, err := sshClient.RunCMD(sendCmd) - outputPID = strings.TrimSpace(outputPID) - if err != nil || strings.HasPrefix(outputPID, "stderr:") { - logger.Errorf("DumpStart err: %s => %s", outputPID, err.Error()) - return "", err - } - - // 检查进程 ps aux | grep tcpdump - // 强杀 sudo pkill tcpdump - pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - dumpPIDMap.Store(pidKey, outputPID) - return taskCode, err -} - -// DumpStop 停止已存在抓包句柄 -func (s *TcpdumpImpl) DumpStop(neType, neId, taskCode string) (string, error) { - // 查询网元获取IP - neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) - if neInfo.NeId != neId || neInfo.IP == "" { - return "", fmt.Errorf("app.common.noNEInfo") - } - // 网元主机的SSH客户端 - sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) - if err != nil { - return "", err - } - defer sshClient.Close() - - // 是否存在执行过的进程 - pidKey := fmt.Sprintf("%s_%s_%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - pid, ok := dumpPIDMap.Load(pidKey) - if !ok || pid == "" { - return "", fmt.Errorf("tcpdump is not running") - } - defer dumpPIDMap.Delete(pidKey) - - // 存放文件目录 /tmp/omc/tcpdump/udm/001/20240817104241 - neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", strings.ToLower(neInfo.NeType), neInfo.NeId, taskCode) - // 命令拼装 - sendCmd := fmt.Sprintf("pids=$(pgrep -P %s) && [ -n \"$pids\" ] && sudo kill $pids;sudo timeout 2s cat %s/tcpdump.log", pid, neDirTemp) - // pids=$(pgrep -P 1914341) && [ -n "$pids" ] && sudo kill $pids;sudo timeout 2s cat tcpdump.log - output, err := sshClient.RunCMD(sendCmd) - if err != nil || strings.HasPrefix(output, "stderr:") { - logger.Warnf("DumpStop err: %s => %s", strings.TrimSpace(output), err.Error()) - return "", err - } - return output, nil -} - -// DumpDownload 抓包文件网元端复制到本地输出zip文件 -func (s *TcpdumpImpl) DumpDownload(neType, neId, taskCode string) (string, error) { - // 查询网元获取IP - neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) - if neInfo.NeId != neId || neInfo.IP == "" { - return "", fmt.Errorf("app.common.noNEInfo") - } - // 网元主机的SSH客户端 - sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) - if err != nil { - return "", err - } - defer sshClient.Close() - // 网元主机的SSH客户端进行文件传输 - sftpClient, err := sshClient.NewClientSFTP() - if err != nil { - return "", fmt.Errorf("ne info sftp client err") - } - defer sftpClient.Close() - - neTypeLower := strings.ToLower(neInfo.NeType) - // 网管本地路径 - localDirPath := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s", neTypeLower, neInfo.NeId) - if runtime.GOOS == "windows" { - localDirPath = fmt.Sprintf("C:%s", localDirPath) - } - - // 网元pcap目录 /tmp/omc/tcpdump/udm/001/20240817104241 - sshClient.RunCMD("mkdir -p /tmp/omc && sudo chmod 777 -R /tmp/omc") - neDirTemp := fmt.Sprintf("/tmp/omc/tcpdump/%s/%s/%s", neTypeLower, neInfo.NeId, taskCode) - // 网元端复制到本地 - localDirFilePath := filepath.Join(localDirPath, taskCode) - if err = sftpClient.CopyDirRemoteToLocal(neDirTemp, localDirFilePath); err != nil { - return "", fmt.Errorf("copy tcpdump file err") - } - - // 压缩zip文件名 - zipFileName := fmt.Sprintf("%s-%s-pcap-%s.zip", neTypeLower, neInfo.NeId, taskCode) - zipFilePath := filepath.Join(localDirPath, zipFileName) - if err := file.CompressZipByDir(zipFilePath, localDirFilePath); err != nil { - return "", fmt.Errorf("compress zip err") - } - - _ = os.RemoveAll(localDirFilePath) // 删除本地临时目录 - return zipFilePath, nil -} - -// UPFTrace UPF标准版内部抓包 -func (s *TcpdumpImpl) UPFTrace(neType, neId, cmdStr string) (string, error) { - // 命令检查 - if strings.Contains(cmdStr, "file") { - return "", fmt.Errorf("command cannot contain file") - } - - // 查询网元获取IP - neInfo := s.neInfoService.SelectNeInfoByNeTypeAndNeID(neType, neId) - if neInfo.NeId != neId || neInfo.IP == "" { - return "", fmt.Errorf("app.common.noNEInfo") - } - // 网元主机的SSH客户端 - sshClient, err := s.neInfoService.NeRunSSHClient(neInfo.NeType, neInfo.NeId) - if err != nil { - return "", err - } - defer sshClient.Close() - // 网元主机的Telnet客户端 - telnetClient, err := s.neInfoService.NeRunTelnetClient("UPF", neInfo.NeId, 2) - if err != nil { - return "", err - } - defer telnetClient.Close() - - // 命令拼装 - fileName := fmt.Sprintf("%s_%s_part_%s.pcap ", neInfo.NeType, neInfo.NeId, time.Now().Format("20060102150405")) - pcapCmd := fmt.Sprintf("%s\r\n", cmdStr) - // 以off结尾是停止抓包，不需要写文件 - if !strings.Contains(cmdStr, "off") { - // pcap trace rx tx max 100000 intfc any file UPF_001_part_20240817164516.pcap - pcapCmd = fmt.Sprintf("%s file %s\r\n", cmdStr, fileName) - } - // 发送命令 UPF内部默认输出路径/tmp只能写文件名 - // pcap trace rx tx max 100000 intfc any file upf_test.pcap - // pcap trace rx tx off - output, err := telnetClient.RunCMD(pcapCmd) - if err != nil { - logger.Warnf("DumpUPF err: %s => %s", output, err.Error()) - return "", err - } - - // 结果截取 - arr := strings.Split(output, "\r\n") - if len(arr) == 2 { - return "", fmt.Errorf("trace pacp run failed") - } - if len(arr) > 3 { - resMsg := arr[2] - // pcap trace: unknown input `f file UPF_001_part_2024-08-19...' - // pcap trace: dispatch trace already enabled... - // pcap trace: dispatch trace already disabled... - // pcap trace: No packets captured... - // Write 100000 packets to /tmp/UPF_001_part_20240817164516.pcap, and stop capture... - if strings.Contains(resMsg, "unknown input") { - return "", fmt.Errorf("trace pacp unknown input") - } - if strings.Contains(resMsg, "already enabled") { - return "", fmt.Errorf("trace pacp already running") - } - if strings.Contains(resMsg, "already disabled") { - return "", fmt.Errorf("trace pacp not running") - } - if strings.Contains(resMsg, "No packets") { - return "", fmt.Errorf("trace pacp not packets") - } - if strings.Contains(resMsg, "packets to") { - matches := regexp.MustCompile(`(/tmp/[^,\s]+)`).FindStringSubmatch(resMsg) - if len(matches) == 0 { - return "", fmt.Errorf("file path not found") - } - return matches[0], nil - } - } - return "trace pacp running", nil -}