diff --git a/userbenchmark/release-test/monitor_proc.sh b/userbenchmark/release-test/monitor_proc.sh index d6e1c04f05..0c06498ed8 100644 --- a/userbenchmark/release-test/monitor_proc.sh +++ b/userbenchmark/release-test/monitor_proc.sh @@ -16,12 +16,12 @@ get_gpu_max_memory_usage_cuda() { local my_pid=$1 local max=$2 local curr - # Some processes might not use the GPU - if ! nvidia-smi dmon -s m -c 1 -o T -i 0 | grep "${my_pid}" >/dev/null 2>/dev/null; then + curr=$(nvidia-smi dmon -s m -c 1 -o T -i 0 | tail -n 1 | awk '{print $4}' | sort | tail -1 | grep -o "[0-9.]*") + # Some processes might not use the GPU, then memory usage should be 0 + if [ "${curr}" -eq 0 ] ; then echo "${max}" return fi - curr=$(nvidia-smi dmon -s m -c 1 -o T -i 0 | grep "${my_pid}" | awk '{print $4}' | sort | tail -1 | grep -o "[0-9.]*") max "${curr}" "${max}" }